From 5a1bb0b12e0c104fd575aa2934bc8647071c0959 Mon Sep 17 00:00:00 2001 From: psolstice Date: Sat, 17 Aug 2024 17:04:03 +0200 Subject: [PATCH] Fix warnings in build process (#1470) * Update immer library to v0.8.1 (5875f7739a6c642ad58cbedadb509c86d421217e) * Fixed warnings in Linux / GCC. Fixed warnings in Linux / Clang. Fixed warnings in Win cross-compile. no-gui build only. * Fixed missing immer headers during the build * Fixed Qt compilation warnings --------- Co-authored-by: aleflm --- src/Makefile.am | 104 +- src/compat/glibcxx_sanity.cpp | 2 +- src/immer/algorithm.hpp | 155 +- src/immer/array.hpp | 189 +- src/immer/array_transient.hpp | 79 +- src/immer/atom.hpp | 93 +- src/immer/box.hpp | 134 +- src/immer/config.hpp | 83 +- src/immer/detail/arrays/no_capacity.hpp | 94 +- src/immer/detail/arrays/node.hpp | 69 +- src/immer/detail/arrays/with_capacity.hpp | 168 +- src/immer/detail/combine_standard_layout.hpp | 92 +- src/immer/detail/hamts/bits.hpp | 116 +- src/immer/detail/hamts/champ.hpp | 1410 ++++++++++++-- src/immer/detail/hamts/champ_iterator.hpp | 50 +- src/immer/detail/hamts/node.hpp | 930 +++++++--- src/immer/detail/iterator_facade.hpp | 93 +- src/immer/detail/rbts/bits.hpp | 5 +- src/immer/detail/rbts/node.hpp | 671 ++++--- src/immer/detail/rbts/operations.hpp | 1612 +++++++++-------- src/immer/detail/rbts/position.hpp | 944 ++++++---- src/immer/detail/rbts/rbtree.hpp | 319 ++-- src/immer/detail/rbts/rbtree_iterator.hpp | 37 +- src/immer/detail/rbts/rrbtree.hpp | 1042 ++++++----- src/immer/detail/rbts/rrbtree_iterator.hpp | 32 +- src/immer/detail/rbts/visitor.hpp | 10 +- src/immer/detail/ref_count_base.hpp | 2 +- src/immer/detail/type_traits.hpp | 232 +-- src/immer/detail/util.hpp | 265 ++- .../experimental/detail/dvektor_impl.hpp | 205 +-- src/immer/experimental/dvektor.hpp | 36 +- src/immer/flex_vector.hpp | 330 ++-- src/immer/flex_vector_transient.hpp | 114 +- src/immer/heap/cpp_heap.hpp | 3 +- src/immer/heap/debug_size_heap.hpp | 27 +- src/immer/heap/free_list_heap.hpp | 3 +- src/immer/heap/free_list_node.hpp | 3 +- src/immer/heap/gc_heap.hpp | 29 +- src/immer/heap/heap_policy.hpp | 46 +- src/immer/heap/malloc_heap.hpp | 11 +- src/immer/heap/split_heap.hpp | 6 +- src/immer/heap/tags.hpp | 3 +- .../heap/thread_local_free_list_heap.hpp | 12 +- src/immer/heap/unsafe_free_list_heap.hpp | 23 +- src/immer/lock/no_lock_policy.hpp | 25 + src/immer/lock/spinlock_policy.hpp | 68 + src/immer/map.hpp | 369 +++- src/immer/map_transient.hpp | 301 ++- src/immer/memory_policy.hpp | 52 +- src/immer/refcount/no_refcount_policy.hpp | 20 +- src/immer/refcount/refcount_policy.hpp | 89 +- src/immer/refcount/unsafe_refcount_policy.hpp | 10 +- src/immer/set.hpp | 196 +- src/immer/set_transient.hpp | 168 +- src/immer/table.hpp | 547 ++++++ src/immer/table_transient.hpp | 281 +++ src/immer/transience/gc_transience_policy.hpp | 11 +- src/immer/transience/no_transience_policy.hpp | 8 +- src/immer/vector.hpp | 193 +- src/immer/vector_transient.hpp | 79 +- src/libspark/bpplus.cpp | 2 +- src/qt/addresstablemodel.cpp | 14 +- src/qt/bantablemodel.cpp | 6 +- src/qt/bitcoin.cpp | 2 +- src/qt/bitcoinamountfield.cpp | 5 +- src/qt/bitcoingui.cpp | 2 +- src/qt/guiutil.cpp | 38 +- src/qt/guiutil.h | 44 + src/qt/lelantusdialog.cpp | 12 +- src/qt/notifymnemonic.cpp | 2 +- src/qt/optionsmodel.cpp | 16 +- src/qt/overviewpage.cpp | 2 +- src/qt/pcodemodel.cpp | 4 +- src/qt/peertablemodel.cpp | 4 +- src/qt/receivecoinsdialog.cpp | 2 +- src/qt/receiverequestdialog.cpp | 12 +- src/qt/recentrequeststablemodel.cpp | 2 +- src/qt/recentrequeststablemodel.h | 2 +- src/qt/sendcoinsdialog.cpp | 12 +- src/qt/splashscreen.h | 2 +- src/qt/transactiontablemodel.cpp | 4 +- src/qt/transactionview.cpp | 18 +- src/qt/utilitydialog.h | 2 +- src/rpc/net.cpp | 2 +- src/rpc/protocol.cpp | 2 +- src/script/ismine.cpp | 2 + src/secp256k1/src/ecmult.h | 1 + src/secp256k1/src/ecmult_const_impl.h | 7 - src/secp256k1/src/ecmult_impl.h | 1 - src/secp256k1/src/scratch_impl.h | 1 + src/util.cpp | 4 +- src/wallet/wallet.cpp | 6 +- 92 files changed, 8576 insertions(+), 3959 deletions(-) create mode 100644 src/immer/lock/no_lock_policy.hpp create mode 100644 src/immer/lock/spinlock_policy.hpp create mode 100644 src/immer/table.hpp create mode 100644 src/immer/table_transient.hpp diff --git a/src/Makefile.am b/src/Makefile.am index 3f43272474..ba939bc51f 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -267,63 +267,77 @@ BITCOIN_CORE_H = \ llmq/quorums_chainlocks.h\ llmq/quorums_init.h \ llmq/quorums_signing_shares.h \ - immer/atom.hpp \ - immer/set.hpp \ - immer/box.hpp \ - immer/flex_vector_transient.hpp \ - immer/transience/no_transience_policy.hpp \ - immer/transience/gc_transience_policy.hpp \ + immer/array_transient.hpp \ immer/map_transient.hpp \ - immer/set_transient.hpp \ - immer/flex_vector.hpp \ - immer/refcount/enable_intrusive_ptr.hpp \ - immer/refcount/no_refcount_policy.hpp \ + immer/map.hpp \ + immer/lock \ + immer/lock/no_lock_policy.hpp \ + immer/lock/spinlock_policy.hpp \ + immer/heap \ + immer/heap/heap_policy.hpp \ + immer/heap/thread_local_free_list_heap.hpp \ + immer/heap/tags.hpp \ + immer/heap/split_heap.hpp \ + immer/heap/with_data.hpp \ + immer/heap/unsafe_free_list_heap.hpp \ + immer/heap/cpp_heap.hpp \ + immer/heap/malloc_heap.hpp \ + immer/heap/gc_heap.hpp \ + immer/heap/debug_size_heap.hpp \ + immer/heap/free_list_heap.hpp \ + immer/heap/identity_heap.hpp \ + immer/heap/free_list_node.hpp \ + immer/set.hpp \ + immer/vector_transient.hpp \ + immer/atom.hpp \ + immer/config.hpp \ + immer/experimental \ + immer/experimental/dvektor.hpp \ + immer/experimental/detail \ + immer/experimental/detail/dvektor_impl.hpp \ + immer/algorithm.hpp \ + immer/table.hpp \ + immer/memory_policy.hpp \ + immer/refcount \ immer/refcount/unsafe_refcount_policy.hpp \ + immer/refcount/no_refcount_policy.hpp \ + immer/refcount/enable_intrusive_ptr.hpp \ immer/refcount/refcount_policy.hpp \ - immer/memory_policy.hpp \ - immer/config.hpp \ - immer/array.hpp \ + immer/transience \ + immer/transience/gc_transience_policy.hpp \ + immer/transience/no_transience_policy.hpp \ + immer/flex_vector.hpp \ + immer/box.hpp \ immer/vector.hpp \ - immer/detail/arrays/no_capacity.hpp \ - immer/detail/arrays/node.hpp \ - immer/detail/arrays/with_capacity.hpp \ - immer/detail/type_traits.hpp \ - immer/detail/rbts/rbtree_iterator.hpp \ + immer/array.hpp \ + immer/set_transient.hpp \ + immer/detail \ + immer/detail/rbts \ + immer/detail/rbts/operations.hpp \ immer/detail/rbts/rbtree.hpp \ - immer/detail/rbts/rrbtree.hpp \ + immer/detail/rbts/rbtree_iterator.hpp \ immer/detail/rbts/visitor.hpp \ + immer/detail/rbts/rrbtree_iterator.hpp \ + immer/detail/rbts/rrbtree.hpp \ + immer/detail/rbts/node.hpp \ immer/detail/rbts/position.hpp \ immer/detail/rbts/bits.hpp \ - immer/detail/rbts/node.hpp \ - immer/detail/rbts/operations.hpp \ - immer/detail/rbts/rrbtree_iterator.hpp \ - immer/detail/combine_standard_layout.hpp \ immer/detail/ref_count_base.hpp \ - immer/detail/hamts/champ_iterator.hpp \ + immer/detail/arrays \ + immer/detail/arrays/with_capacity.hpp \ + immer/detail/arrays/node.hpp \ + immer/detail/arrays/no_capacity.hpp \ + immer/detail/util.hpp \ + immer/detail/hamts \ immer/detail/hamts/champ.hpp \ - immer/detail/hamts/bits.hpp \ + immer/detail/hamts/champ_iterator.hpp \ immer/detail/hamts/node.hpp \ - immer/detail/util.hpp \ + immer/detail/hamts/bits.hpp \ + immer/detail/type_traits.hpp \ immer/detail/iterator_facade.hpp \ - immer/vector_transient.hpp \ - immer/experimental/dvektor.hpp \ - immer/experimental/detail/dvektor_impl.hpp \ - immer/heap/free_list_heap.hpp \ - immer/heap/tags.hpp \ - immer/heap/debug_size_heap.hpp \ - immer/heap/heap_policy.hpp \ - immer/heap/gc_heap.hpp \ - immer/heap/unsafe_free_list_heap.hpp \ - immer/heap/split_heap.hpp \ - immer/heap/identity_heap.hpp \ - immer/heap/thread_local_free_list_heap.hpp \ - immer/heap/malloc_heap.hpp \ - immer/heap/cpp_heap.hpp \ - immer/heap/free_list_node.hpp \ - immer/heap/with_data.hpp \ - immer/algorithm.hpp \ - immer/array_transient.hpp \ - immer/map.hpp + immer/detail/combine_standard_layout.hpp \ + immer/flex_vector_transient.hpp \ + immer/table_transient.hpp obj/build.h: FORCE @$(MKDIR_P) $(builddir)/obj diff --git a/src/compat/glibcxx_sanity.cpp b/src/compat/glibcxx_sanity.cpp index cee8a98c7f..5c4e5e6378 100644 --- a/src/compat/glibcxx_sanity.cpp +++ b/src/compat/glibcxx_sanity.cpp @@ -47,7 +47,7 @@ bool sanity_test_range_fmt() { std::string test; try { - test.at(1); + (void) test.at(1); } catch (const std::out_of_range&) { return true; } catch (...) { diff --git a/src/immer/algorithm.hpp b/src/immer/algorithm.hpp index df9ff28a83..382df02dbc 100644 --- a/src/immer/algorithm.hpp +++ b/src/immer/algorithm.hpp @@ -9,6 +9,8 @@ #pragma once #include +#include +#include #include #include @@ -47,8 +49,8 @@ template void for_each_chunk(const Iterator& first, const Iterator& last, Fn&& fn) { assert(&first.impl() == &last.impl()); - first.impl().for_each_chunk(first.index(), last.index(), - std::forward(fn)); + first.impl().for_each_chunk( + first.index(), last.index(), std::forward(fn)); } template @@ -81,8 +83,8 @@ template bool for_each_chunk_p(const Iterator& first, const Iterator& last, Fn&& fn) { assert(&first.impl() == &last.impl()); - return first.impl().for_each_chunk_p(first.index(), last.index(), - std::forward(fn)); + return first.impl().for_each_chunk_p( + first.index(), last.index(), std::forward(fn)); } template @@ -91,14 +93,34 @@ bool for_each_chunk_p(const T* first, const T* last, Fn&& fn) return std::forward(fn)(first, last); } +namespace detail { + +template +T accumulate_move(Iter first, Iter last, T init) +{ + for (; first != last; ++first) + init = std::move(init) + *first; + return init; +} + +template +T accumulate_move(Iter first, Iter last, T init, Fn op) +{ + for (; first != last; ++first) + init = op(std::move(init), *first); + return init; +} + +} // namespace detail + /*! * Equivalent of `std::accumulate` applied to the range `r`. */ template T accumulate(Range&& r, T init) { - for_each_chunk(r, [&] (auto first, auto last) { - init = std::accumulate(first, last, init); + for_each_chunk(r, [&](auto first, auto last) { + init = detail::accumulate_move(first, last, init); }); return init; } @@ -106,8 +128,8 @@ T accumulate(Range&& r, T init) template T accumulate(Range&& r, T init, Fn fn) { - for_each_chunk(r, [&] (auto first, auto last) { - init = std::accumulate(first, last, init, fn); + for_each_chunk(r, [&](auto first, auto last) { + init = detail::accumulate_move(first, last, init, fn); }); return init; } @@ -119,8 +141,8 @@ T accumulate(Range&& r, T init, Fn fn) template T accumulate(Iterator first, Iterator last, T init) { - for_each_chunk(first, last, [&] (auto first, auto last) { - init = std::accumulate(first, last, init); + for_each_chunk(first, last, [&](auto first, auto last) { + init = detail::accumulate_move(first, last, init); }); return init; } @@ -128,8 +150,8 @@ T accumulate(Iterator first, Iterator last, T init) template T accumulate(Iterator first, Iterator last, T init, Fn fn) { - for_each_chunk(first, last, [&] (auto first, auto last) { - init = std::accumulate(first, last, init, fn); + for_each_chunk(first, last, [&](auto first, auto last) { + init = detail::accumulate_move(first, last, init, fn); }); return init; } @@ -140,7 +162,7 @@ T accumulate(Iterator first, Iterator last, T init, Fn fn) template Fn&& for_each(Range&& r, Fn&& fn) { - for_each_chunk(r, [&] (auto first, auto last) { + for_each_chunk(r, [&](auto first, auto last) { for (; first != last; ++first) fn(*first); }); @@ -154,7 +176,7 @@ Fn&& for_each(Range&& r, Fn&& fn) template Fn&& for_each(Iterator first, Iterator last, Fn&& fn) { - for_each_chunk(first, last, [&] (auto first, auto last) { + for_each_chunk(first, last, [&](auto first, auto last) { for (; first != last; ++first) fn(*first); }); @@ -167,9 +189,8 @@ Fn&& for_each(Iterator first, Iterator last, Fn&& fn) template OutIter copy(Range&& r, OutIter out) { - for_each_chunk(r, [&] (auto first, auto last) { - out = std::copy(first, last, out); - }); + for_each_chunk( + r, [&](auto first, auto last) { out = std::copy(first, last, out); }); return out; } @@ -180,7 +201,7 @@ OutIter copy(Range&& r, OutIter out) template OutIter copy(InIter first, InIter last, OutIter out) { - for_each_chunk(first, last, [&] (auto first, auto last) { + for_each_chunk(first, last, [&](auto first, auto last) { out = std::copy(first, last, out); }); return out; @@ -192,9 +213,8 @@ OutIter copy(InIter first, InIter last, OutIter out) template bool all_of(Range&& r, Pred p) { - return for_each_chunk_p(r, [&] (auto first, auto last) { - return std::all_of(first, last, p); - }); + return for_each_chunk_p( + r, [&](auto first, auto last) { return std::all_of(first, last, p); }); } /*! @@ -204,11 +224,102 @@ bool all_of(Range&& r, Pred p) template bool all_of(Iter first, Iter last, Pred p) { - return for_each_chunk_p(first, last, [&] (auto first, auto last) { + return for_each_chunk_p(first, last, [&](auto first, auto last) { return std::all_of(first, last, p); }); } +/*! + * Object that can be used to process changes as computed by the @a diff + * algorithm. + * + * @tparam AddedFn Unary function that is be called whenever an added element is + * found. It is called with the added element as argument. + * + * @tparam RemovedFn Unary function that is called whenever a removed element is + * found. It is called with the removed element as argument. + * + * @tparam ChangedFn Unary function that is called whenever a changed element is + * found. It is called with the changed element as argument. + */ +template +struct differ +{ + AddedFn added; + RemovedFn removed; + ChangedFn changed; +}; + +/*! + * Produces a @a differ object with `added`, `removed` and `changed` functions. + */ +template +auto make_differ(AddedFn&& added, RemovedFn&& removed, ChangedFn&& changed) + -> differ, + std::decay_t, + std::decay_t> +{ + return {std::forward(added), + std::forward(removed), + std::forward(changed)}; +} + +/*! + * Produces a @a differ object with `added` and `removed` functions and no + * `changed` function. + */ +template +auto make_differ(AddedFn&& added, RemovedFn&& removed) +{ + return make_differ(std::forward(added), + std::forward(removed), + [](auto&&...) {}); +} + +/*! + * Compute the differences between `a` and `b`. + * + * Changes detected are notified via the differ object, which should support the + * following expressions: + * + * - `differ.added(x)`, invoked when element `x` is found in `b` but not in + * `a`. + * + * - `differ.removed(x)`, invoked when element `x` is found in `a` but not in + * `b`. + * + * - `differ.changed(x, y)`, invoked when element `x` and `y` from `a` and `b` + * share the same key but map to a different value. + * + * This method leverages structural sharing to offer a complexity @f$ O(|diff|) + * @f$ when `b` is derived from `a` by performing @f$ |diff| @f$ updates. This + * is, this function can detect changes in effectively constant time per update, + * as oposed to the @f$ O(|a|+|b|) @f$ complexity of a trivial implementation. + * + * @rst + * + * .. note:: This method is only implemented for ``map`` and ``set``. When sets + * are diffed, the ``changed`` function is never called. + * + * @endrst + */ +template +void diff(const T& a, const T& b, Differ&& differ) +{ + a.impl().template diff>( + b.impl(), std::forward(differ)); +} + +/*! + * Compute the differences between `a` and `b` using the callbacks in `fns` as + * differ. Equivalent to `diff(a, b, make_differ(fns)...)`. + */ +template +void diff(const T& a, const T& b, Fns&&... fns) +{ + diff(a, b, make_differ(std::forward(fns)...)); +} + /** @} */ // group: algorithm } // namespace immer diff --git a/src/immer/array.hpp b/src/immer/array.hpp index 0f73649fb3..f71477c239 100644 --- a/src/immer/array.hpp +++ b/src/immer/array.hpp @@ -8,8 +8,10 @@ #pragma once -#include #include +#include + +#include namespace immer { @@ -31,7 +33,7 @@ class array_transient; * .. tip:: Don't be fooled by the bad complexity of this data * structure. It is a great choice for short sequence or when it * is seldom or never changed. This depends on the ``sizeof(T)`` - * and the expensiveness of its ``T``'s copy constructor, in case + * and the expensiveness of its ``T``'s copy constructor. In case * of doubt, measure. For basic types, using an `array` when * :math:`n < 100` is a good heuristic. * @@ -40,18 +42,18 @@ class array_transient; template class array { - using impl_t = std::conditional_t< - MemoryPolicy::use_transient_rvalues, - detail::arrays::with_capacity, - detail::arrays::no_capacity>; + using impl_t = + std::conditional_t, + detail::arrays::no_capacity>; using move_t = std::integral_constant; public: - using value_type = T; - using reference = const T&; - using size_type = std::size_t; + using value_type = T; + using reference = const T&; + using size_type = std::size_t; using difference_type = std::ptrdiff_t; using const_reference = const T&; @@ -79,16 +81,17 @@ class array * Constructs a array containing the elements in the range * defined by the forward iterator `first` and range sentinel `last`. */ - template - && detail::is_forward_iterator_v, bool> = true> + template && + detail::is_forward_iterator_v, + bool> = true> array(Iter first, Sent last) : impl_{impl_t::from_range(first, last)} {} /*! - * Constructs a array containing the element `val` repeated `n` + * Constructs an array containing the element `val` repeated `n` * times. */ array(size_type n, T v = {}) @@ -100,63 +103,72 @@ class array * collection. It does not allocate memory and its complexity is * @f$ O(1) @f$. */ - iterator begin() const { return impl_.data(); } + IMMER_NODISCARD iterator begin() const { return impl_.data(); } /*! * Returns an iterator pointing just after the last element of the - * collection. It does not allocate and its complexity is @f$ O(1) @f$. + * collection. It does not allocate memory and its complexity is @f$ O(1) + * @f$. */ - iterator end() const { return impl_.data() + impl_.size; } + IMMER_NODISCARD iterator end() const { return impl_.data() + impl_.size; } /*! * Returns an iterator that traverses the collection backwards, * pointing at the first element of the reversed collection. It * does not allocate memory and its complexity is @f$ O(1) @f$. */ - reverse_iterator rbegin() const { return reverse_iterator{end()}; } + IMMER_NODISCARD reverse_iterator rbegin() const + { + return reverse_iterator{end()}; + } /*! * Returns an iterator that traverses the collection backwards, * pointing after the last element of the reversed collection. It * does not allocate memory and its complexity is @f$ O(1) @f$. */ - reverse_iterator rend() const { return reverse_iterator{begin()}; } + IMMER_NODISCARD reverse_iterator rend() const + { + return reverse_iterator{begin()}; + } /*! * Returns the number of elements in the container. It does * not allocate memory and its complexity is @f$ O(1) @f$. */ - std::size_t size() const { return impl_.size; } + IMMER_NODISCARD std::size_t size() const { return impl_.size; } /*! * Returns `true` if there are no elements in the container. It * does not allocate memory and its complexity is @f$ O(1) @f$. */ - bool empty() const { return impl_.d->empty(); } + IMMER_NODISCARD bool empty() const { return impl_.size == 0; } /*! * Access the raw data. */ - const T* data() const { return impl_.data(); } + IMMER_NODISCARD const T* data() const { return impl_.data(); } /*! * Access the last element. */ - const T& back() const { return data()[size() - 1]; } + IMMER_NODISCARD const T& back() const { return data()[size() - 1]; } /*! * Access the first element. */ - const T& front() const { return data()[0]; } + IMMER_NODISCARD const T& front() const { return data()[0]; } /*! * Returns a `const` reference to the element at position `index`. - * It is undefined when @f$ 0 index \geq size() @f$. It does not + * It is undefined when @f$ index \geq size() @f$. It does not * allocate memory and its complexity is *effectively* @f$ O(1) * @f$. */ - reference operator[] (size_type index) const - { return impl_.get(index); } + IMMER_NODISCARD reference operator[](size_type index) const + { + return impl_.get(index); + } /*! * Returns a `const` reference to the element at position @@ -164,16 +176,19 @@ class array * index \geq size() @f$. It does not allocate memory and its * complexity is *effectively* @f$ O(1) @f$. */ - reference at(size_type index) const - { return impl_.get_check(index); } + reference at(size_type index) const { return impl_.get_check(index); } /*! * Returns whether the vectors are equal. */ - bool operator==(const array& other) const - { return impl_.equals(other.impl_); } - bool operator!=(const array& other) const - { return !(*this == other); } + IMMER_NODISCARD bool operator==(const array& other) const + { + return impl_.equals(other.impl_); + } + IMMER_NODISCARD bool operator!=(const array& other) const + { + return !(*this == other); + } /*! * Returns an array with `value` inserted at the end. It may @@ -190,11 +205,15 @@ class array * * @endrst */ - array push_back(value_type value) const& - { return impl_.push_back(std::move(value)); } + IMMER_NODISCARD array push_back(value_type value) const& + { + return impl_.push_back(std::move(value)); + } - decltype(auto) push_back(value_type value) && - { return push_back_move(move_t{}, std::move(value)); } + IMMER_NODISCARD decltype(auto) push_back(value_type value) && + { + return push_back_move(move_t{}, std::move(value)); + } /*! * Returns an array containing value `value` at position `idx`. @@ -212,11 +231,15 @@ class array * * @endrst */ - array set(std::size_t index, value_type value) const& - { return impl_.assoc(index, std::move(value)); } + IMMER_NODISCARD array set(std::size_t index, value_type value) const& + { + return impl_.assoc(index, std::move(value)); + } - decltype(auto) set(size_type index, value_type value) && - { return set_move(move_t{}, index, std::move(value)); } + IMMER_NODISCARD decltype(auto) set(size_type index, value_type value) && + { + return set_move(move_t{}, index, std::move(value)); + } /*! * Returns an array containing the result of the expression @@ -236,12 +259,16 @@ class array * @endrst */ template - array update(std::size_t index, FnT&& fn) const& - { return impl_.update(index, std::forward(fn)); } + IMMER_NODISCARD array update(std::size_t index, FnT&& fn) const& + { + return impl_.update(index, std::forward(fn)); + } template - decltype(auto) update(size_type index, FnT&& fn) && - { return update_move(move_t{}, index, std::forward(fn)); } + IMMER_NODISCARD decltype(auto) update(size_type index, FnT&& fn) && + { + return update_move(move_t{}, index, std::forward(fn)); + } /*! * Returns a array containing only the first `min(elems, size())` @@ -259,20 +286,36 @@ class array * * @endrst */ - array take(size_type elems) const& - { return impl_.take(elems); } + IMMER_NODISCARD array take(size_type elems) const& + { + return impl_.take(elems); + } - decltype(auto) take(size_type elems) && - { return take_move(move_t{}, elems); } + IMMER_NODISCARD decltype(auto) take(size_type elems) && + { + return take_move(move_t{}, elems); + } /*! * Returns an @a transient form of this container, an * `immer::array_transient`. */ - transient_type transient() const& - { return transient_type{ impl_ }; } - transient_type transient() && - { return transient_type{ std::move(impl_) }; } + IMMER_NODISCARD transient_type transient() const& + { + return transient_type{impl_}; + } + IMMER_NODISCARD transient_type transient() && + { + return transient_type{std::move(impl_)}; + } + + /*! + * Returns a value that can be used as identity for the container. If two + * values have the same identity, they are guaranteed to be equal and to + * contain the same objects. However, two equal containers are not + * guaranteed to have the same identity. + */ + void* identity() const { return impl_.ptr; } // Semi-private const impl_t& impl() const { return impl_; } @@ -280,29 +323,51 @@ class array private: friend transient_type; - array(impl_t impl) : impl_(std::move(impl)) {} + array(impl_t impl) + : impl_(std::move(impl)) + {} array&& push_back_move(std::true_type, value_type value) - { impl_.push_back_mut({}, std::move(value)); return std::move(*this); } + { + impl_.push_back_mut({}, std::move(value)); + return std::move(*this); + } array push_back_move(std::false_type, value_type value) - { return impl_.push_back(std::move(value)); } + { + return impl_.push_back(std::move(value)); + } array&& set_move(std::true_type, size_type index, value_type value) - { impl_.assoc_mut({}, index, std::move(value)); return std::move(*this); } + { + impl_.assoc_mut({}, index, std::move(value)); + return std::move(*this); + } array set_move(std::false_type, size_type index, value_type value) - { return impl_.assoc(index, std::move(value)); } + { + return impl_.assoc(index, std::move(value)); + } template array&& update_move(std::true_type, size_type index, Fn&& fn) - { impl_.update_mut({}, index, std::forward(fn)); return std::move(*this); } + { + impl_.update_mut({}, index, std::forward(fn)); + return std::move(*this); + } template array update_move(std::false_type, size_type index, Fn&& fn) - { return impl_.update(index, std::forward(fn)); } + { + return impl_.update(index, std::forward(fn)); + } array&& take_move(std::true_type, size_type elems) - { impl_.take_mut({}, elems); return std::move(*this); } + { + impl_.take_mut({}, elems); + return std::move(*this); + } array take_move(std::false_type, size_type elems) - { return impl_.take(elems); } + { + return impl_.take(elems); + } impl_t impl_ = impl_t::empty(); }; diff --git a/src/immer/array_transient.hpp b/src/immer/array_transient.hpp index 0084e47ddd..08000b2d92 100644 --- a/src/immer/array_transient.hpp +++ b/src/immer/array_transient.hpp @@ -8,8 +8,10 @@ #pragma once -#include #include +#include + +#include namespace immer { @@ -27,17 +29,16 @@ class array; * @endrst */ template -class array_transient - : MemoryPolicy::transience_t::owner +class array_transient : MemoryPolicy::transience_t::owner { - using impl_t = detail::arrays::with_capacity; + using impl_t = detail::arrays::with_capacity; using impl_no_capacity_t = detail::arrays::no_capacity; - using owner_t = typename MemoryPolicy::transience_t::owner; + using owner_t = typename MemoryPolicy::transience_t::owner; public: - using value_type = T; - using reference = const T&; - using size_type = std::size_t; + using value_type = T; + using reference = const T&; + using size_type = std::size_t; using difference_type = std::ptrdiff_t; using const_reference = const T&; @@ -60,54 +61,65 @@ class array_transient * collection. It does not allocate memory and its complexity is * @f$ O(1) @f$. */ - iterator begin() const { return impl_.data(); } + IMMER_NODISCARD iterator begin() const { return impl_.data(); } /*! * Returns an iterator pointing just after the last element of the * collection. It does not allocate and its complexity is @f$ O(1) @f$. */ - iterator end() const { return impl_.data() + impl_.size; } + IMMER_NODISCARD iterator end() const { return impl_.data() + impl_.size; } /*! * Returns an iterator that traverses the collection backwards, * pointing at the first element of the reversed collection. It * does not allocate memory and its complexity is @f$ O(1) @f$. */ - reverse_iterator rbegin() const { return reverse_iterator{end()}; } + IMMER_NODISCARD reverse_iterator rbegin() const + { + return reverse_iterator{end()}; + } /*! * Returns an iterator that traverses the collection backwards, * pointing after the last element of the reversed collection. It * does not allocate memory and its complexity is @f$ O(1) @f$. */ - reverse_iterator rend() const { return reverse_iterator{begin()}; } + IMMER_NODISCARD reverse_iterator rend() const + { + return reverse_iterator{begin()}; + } /*! * Returns the number of elements in the container. It does * not allocate memory and its complexity is @f$ O(1) @f$. */ - std::size_t size() const { return impl_.size; } + IMMER_NODISCARD std::size_t size() const { return impl_.size; } /*! * Returns `true` if there are no elements in the container. It * does not allocate memory and its complexity is @f$ O(1) @f$. */ - bool empty() const { return impl_.d->empty(); } + IMMER_NODISCARD bool empty() const { return impl_.size == 0; } /*! * Access the raw data. */ - const T* data() const { return impl_.data(); } + IMMER_NODISCARD const T* data() const { return impl_.data(); } + + /*! + * Provide mutable access to the raw underlaying data. + */ + IMMER_NODISCARD T* data_mut() { return impl_.data_mut(*this); } /*! * Access the last element. */ - const T& back() const { return data()[size() - 1]; } + IMMER_NODISCARD const T& back() const { return data()[size() - 1]; } /*! * Access the first element. */ - const T& front() const { return data()[0]; } + IMMER_NODISCARD const T& front() const { return data()[0]; } /*! * Returns a `const` reference to the element at position `index`. @@ -115,8 +127,7 @@ class array_transient * allocate memory and its complexity is *effectively* @f$ O(1) * @f$. */ - reference operator[] (size_type index) const - { return impl_.get(index); } + reference operator[](size_type index) const { return impl_.get(index); } /*! * Returns a `const` reference to the element at position @@ -124,15 +135,16 @@ class array_transient * index \geq size() @f$. It does not allocate memory and its * complexity is *effectively* @f$ O(1) @f$. */ - reference at(size_type index) const - { return impl_.get_check(index); } + reference at(size_type index) const { return impl_.get_check(index); } /*! * Inserts `value` at the end. It may allocate memory and its * complexity is *effectively* @f$ O(1) @f$. */ void push_back(value_type value) - { impl_.push_back_mut(*this, std::move(value)); } + { + impl_.push_back_mut(*this, std::move(value)); + } /*! * Sets to the value `value` at position `idx`. @@ -141,7 +153,9 @@ class array_transient * *effectively* @f$ O(1) @f$. */ void set(size_type index, value_type value) - { impl_.assoc_mut(*this, index, std::move(value)); } + { + impl_.assoc_mut(*this, index, std::move(value)); + } /*! * Updates the array to contain the result of the expression @@ -152,27 +166,30 @@ class array_transient */ template void update(size_type index, FnT&& fn) - { impl_.update_mut(*this, index, std::forward(fn)); } + { + impl_.update_mut(*this, index, std::forward(fn)); + } /*! * Resizes the array to only contain the first `min(elems, size())` * elements. It may allocate memory and its complexity is * *effectively* @f$ O(1) @f$. */ - void take(size_type elems) - { impl_.take_mut(*this, elems); } + void take(size_type elems) { impl_.take_mut(*this, elems); } /*! * Returns an @a immutable form of this container, an * `immer::array`. */ - persistent_type persistent() & + IMMER_NODISCARD persistent_type persistent() & { this->owner_t::operator=(owner_t{}); - return persistent_type{ impl_ }; + return persistent_type{impl_}; + } + IMMER_NODISCARD persistent_type persistent() && + { + return persistent_type{std::move(impl_)}; } - persistent_type persistent() && - { return persistent_type{ std::move(impl_) }; } private: friend persistent_type; @@ -181,7 +198,7 @@ class array_transient : impl_(std::move(impl)) {} - impl_t impl_ = impl_t::empty; + impl_t impl_ = impl_t::empty(); }; } // namespace immer diff --git a/src/immer/atom.hpp b/src/immer/atom.hpp index 206f3c497a..2cf409b916 100644 --- a/src/immer/atom.hpp +++ b/src/immer/atom.hpp @@ -21,14 +21,14 @@ namespace detail { template struct refcount_atom_impl { - using box_type = box; - using value_type = T; + using box_type = box; + using value_type = T; using memory_policy = MemoryPolicy; - using spinlock_t = typename MemoryPolicy::refcount::spinlock_type; - using scoped_lock_t = typename spinlock_t::scoped_lock; + using lock_t = typename MemoryPolicy::lock; + using scoped_lock_t = typename lock_t::scoped_lock; refcount_atom_impl(const refcount_atom_impl&) = delete; - refcount_atom_impl(refcount_atom_impl&&) = delete; + refcount_atom_impl(refcount_atom_impl&&) = delete; refcount_atom_impl& operator=(const refcount_atom_impl&) = delete; refcount_atom_impl& operator=(refcount_atom_impl&&) = delete; @@ -54,7 +54,7 @@ struct refcount_atom_impl scoped_lock_t lock{lock_}; swap(b, impl_); } - return std::move(b); + return b; } template @@ -67,31 +67,30 @@ struct refcount_atom_impl scoped_lock_t lock{lock_}; if (oldv.impl_ == impl_.impl_) { impl_ = newv; - return { newv }; + return {newv}; } } } } private: - mutable spinlock_t lock_; + mutable lock_t lock_; box_type impl_; }; template struct gc_atom_impl { - using box_type = box; - using value_type = T; + using box_type = box; + using value_type = T; using memory_policy = MemoryPolicy; - static_assert( - std::is_same::value, - "gc_atom_impl can only be used when there is no refcount!"); + static_assert(std::is_same::value, + "gc_atom_impl can only be used when there is no refcount!"); gc_atom_impl(const gc_atom_impl&) = delete; - gc_atom_impl(gc_atom_impl&&) = delete; + gc_atom_impl(gc_atom_impl&&) = delete; gc_atom_impl& operator=(const gc_atom_impl&) = delete; gc_atom_impl& operator=(gc_atom_impl&&) = delete; @@ -99,14 +98,11 @@ struct gc_atom_impl : impl_{b.impl_} {} - box_type load() const - { return {impl_.load()}; } + box_type load() const { return {impl_.load()}; } - void store(box_type b) - { impl_.store(b.impl_); } + void store(box_type b) { impl_.store(b.impl_); } - box_type exchange(box_type b) - { return {impl_.exchange(b.impl_)}; } + box_type exchange(box_type b) { return {impl_.exchange(b.impl_)}; } template box_type update(Fn&& fn) @@ -115,7 +111,7 @@ struct gc_atom_impl auto oldv = box_type{impl_.load()}; auto newv = oldv.update(fn); if (impl_.compare_exchange_weak(oldv.impl_, newv.impl_)) - return { newv }; + return {newv}; } } @@ -133,40 +129,39 @@ struct gc_atom_impl * @rst * * .. warning:: If memory policy used includes thread unsafe reference counting, - * no no thread safety is assumed, and the atom becomes thread unsafe too! + * no thread safety is assumed, and the atom becomes thread unsafe too! * - * .. note:: ``box`` provides a value based box of type ``T``, this is, we can - * think about it as a value-based version of ``std::shared_ptr``. In a + * .. note:: ``box`` provides a value based box of type ``T``, this is, we + * can think about it as a value-based version of ``std::shared_ptr``. In a * similar fashion, ``atom`` is in spirit the value-based equivalent of * C++20 ``std::atomic_shared_ptr``. However, the API does not follow * ``std::atomic`` interface closely, since it attempts to be a higher level * construction, most similar to Clojure's ``(atom)``. It is remarkable in * particular that, since ``box`` underlying object is immutable, using - * ``atom`` is fully thread-safe in ways that ``std::atmic_shared_ptr`` is + * ``atom`` is fully thread-safe in ways that ``std::atomic_shared_ptr`` is * not. This is so because dereferencing the underlying pointer in a - * ``std::atomic_share_ptr`` may require further synchronization, in particular - * when invoking non-const methods. + * ``std::atomic_share_ptr`` may require further synchronization, in + * particular when invoking non-const methods. * * @endrst */ -template +template class atom { public: - using box_type = box; - using value_type = T; + using box_type = box; + using value_type = T; using memory_policy = MemoryPolicy; atom(const atom&) = delete; - atom(atom&&) = delete; + atom(atom&&) = delete; void operator=(const atom&) = delete; void operator=(atom&&) = delete; /*! * Constructs an atom holding a value `b`; */ - atom(box_type v={}) + atom(box_type v = {}) : impl_{std::move(v)} {} @@ -182,32 +177,30 @@ class atom /*! * Reads the currently stored value in a thread-safe manner. */ - operator box_type() const - { return impl_.load(); } + operator box_type() const { return impl_.load(); } /*! * Reads the currently stored value in a thread-safe manner. */ - operator value_type() const - { return *impl_.load(); } + operator value_type() const { return *impl_.load(); } /*! * Reads the currently stored value in a thread-safe manner. */ - box_type load() const - { return impl_.load(); } + IMMER_NODISCARD box_type load() const { return impl_.load(); } /*! * Stores a new value in a thread-safe manner. */ - void store(box_type b) - { impl_.store(std::move(b)); } + void store(box_type b) { impl_.store(std::move(b)); } /*! * Stores a new value and returns the old value, in a thread-safe manner. */ - box_type exchange(box_type b) - { return impl_.exchange(std::move(b)); } + IMMER_NODISCARD box_type exchange(box_type b) + { + return impl_.exchange(std::move(b)); + } /*! * Stores the result of applying `fn` to the current value atomically and @@ -223,7 +216,9 @@ class atom */ template box_type update(Fn&& fn) - { return impl_.update(std::forward(fn)); } + { + return impl_.update(std::forward(fn)); + } private: struct get_refcount_atom_impl @@ -248,12 +243,12 @@ class atom // `no_refcount_policy`), we just store the pointer in an atomic. If we use // reference counting, we rely on the reference counting spinlock. using impl_t = typename std::conditional_t< - std::is_same::value, + std::is_same::value, get_gc_atom_impl, - get_refcount_atom_impl - >::template apply::type; + get_refcount_atom_impl>::template apply::type; impl_t impl_; }; -} +} // namespace immer diff --git a/src/immer/box.hpp b/src/immer/box.hpp index b8ad19e1a4..711ac78f09 100644 --- a/src/immer/box.hpp +++ b/src/immer/box.hpp @@ -11,6 +11,8 @@ #include #include +#include + namespace immer { namespace detail { @@ -30,8 +32,7 @@ struct refcount_atom_impl; * operations are never called. Since a box is immutable, copying or * moving just copy the underlying pointers. */ -template +template class box { friend struct detail::gc_atom_impl; @@ -42,51 +43,70 @@ class box T value; template - holder(Args&&... args) : value{std::forward(args)...} {} + holder(Args&&... args) + : value{std::forward(args)...} + {} }; using heap = typename MemoryPolicy::heap::type; holder* impl_ = nullptr; - box(holder* impl) : impl_{impl} {} + box(holder* impl) + : impl_{impl} + {} public: + const holder* impl() const { return impl_; }; + using value_type = T; using memory_policy = MemoryPolicy; /*! * Constructs a box holding `T{}`. */ - box() : impl_{detail::make()} {} + box() + : impl_{detail::make()} + {} /*! * Constructs a box holding `T{arg}` */ template >::value && std::is_constructible::value>> box(Arg&& arg) - : impl_{detail::make(std::forward(arg))} {} + : impl_{detail::make(std::forward(arg))} + {} /*! * Constructs a box holding `T{arg1, arg2, args...}` */ template - box(Arg1&& arg1, Arg2&& arg2, Args&& ...args) - : impl_{detail::make( - std::forward(arg1), - std::forward(arg2), - std::forward(args)...)} + box(Arg1&& arg1, Arg2&& arg2, Args&&... args) + : impl_{detail::make(std::forward(arg1), + std::forward(arg2), + std::forward(args)...)} {} friend void swap(box& a, box& b) - { using std::swap; swap(a.impl_, b.impl_); } + { + using std::swap; + swap(a.impl_, b.impl_); + } box(box&& other) { swap(*this, other); } - box(const box& other) : impl_(other.impl_) { impl_->inc(); } - box& operator=(box&& other) { swap(*this, other); return *this; } + box(const box& other) + : impl_(other.impl_) + { + impl_->inc(); + } + box& operator=(box&& other) + { + swap(*this, other); + return *this; + } box& operator=(const box& other) { auto aux = other; @@ -102,28 +122,16 @@ class box } /*! Query the current value. */ - const T& get() const { return impl_->value; } + IMMER_NODISCARD const T& get() const { return impl_->value; } /*! Conversion to the boxed type. */ operator const T&() const { return get(); } /*! Access via dereference */ - const T& operator* () const { return get(); } + const T& operator*() const { return get(); } /*! Access via pointer member access */ - const T* operator-> () const { return &get(); } - - /*! Comparison. */ - bool operator==(detail::exact_t other) const - { return impl_ == other.value.impl_ || get() == other.value.get(); } - // Note that the `exact_t` disambiguates comparisons against `T{}` - // directly. In that case we want to use `operator T&` and - // compare directly. We definitely never want to convert a value - // to a box (which causes an allocation) just to compare it. - bool operator!=(detail::exact_t other) const - { return !(*this == other.value); } - bool operator<(detail::exact_t other) const - { return get() < other.value.get(); } + const T* operator->() const { return &get(); } /*! * Returns a new box built by applying the `fn` to the underlying @@ -141,12 +149,12 @@ class box * @endrst */ template - box update(Fn&& fn) const& + IMMER_NODISCARD box update(Fn&& fn) const& { return std::forward(fn)(get()); } template - box&& update(Fn&& fn) && + IMMER_NODISCARD box&& update(Fn&& fn) && { if (impl_->unique()) impl_->value = std::forward(fn)(std::move(impl_->value)); @@ -156,6 +164,66 @@ class box } }; +template +IMMER_NODISCARD bool operator==(const box& a, const box& b) +{ + return a.impl() == b.impl() || a.get() == b.get(); +} +template +IMMER_NODISCARD bool operator!=(const box& a, const box& b) +{ + return a.impl() != b.impl() && a.get() != b.get(); +} +template +IMMER_NODISCARD bool operator<(const box& a, const box& b) +{ + return a.impl() != b.impl() && a.get() < b.get(); +} + +template +IMMER_NODISCARD auto operator==(const box& a, T2&& b) + -> std::enable_if_t, std::decay_t>::value, + decltype(a.get() == b)> +{ + return a.get() == b; +} +template +IMMER_NODISCARD auto operator!=(const box& a, T2&& b) + -> std::enable_if_t, std::decay_t>::value, + decltype(a.get() != b)> +{ + return a.get() != b; +} +template +IMMER_NODISCARD auto operator<(const box& a, T2&& b) + -> std::enable_if_t, std::decay_t>::value, + decltype(a.get() < b)> +{ + return a.get() < b; +} + +template +IMMER_NODISCARD auto operator==(T2&& b, const box& a) + -> std::enable_if_t, std::decay_t>::value, + decltype(a.get() == b)> +{ + return a.get() == b; +} +template +IMMER_NODISCARD auto operator!=(T2&& b, const box& a) + -> std::enable_if_t, std::decay_t>::value, + decltype(a.get() != b)> +{ + return a.get() != b; +} +template +IMMER_NODISCARD auto operator<(T2&& b, const box& a) + -> std::enable_if_t, std::decay_t>::value, + decltype(b < a.get())> +{ + return b < a.get(); +} + } // namespace immer namespace std { @@ -163,7 +231,7 @@ namespace std { template struct hash> { - std::size_t operator() (const immer::box& x) const + std::size_t operator()(const immer::box& x) const { return std::hash{}(*x); } diff --git a/src/immer/config.hpp b/src/immer/config.hpp index 67697ef36b..2e8378f6d4 100644 --- a/src/immer/config.hpp +++ b/src/immer/config.hpp @@ -8,6 +8,60 @@ #pragma once +#if (__cplusplus >= 201703L) || (defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) +#define IMMER_HAS_CPP17 1 +#endif + +#if defined(__has_cpp_attribute) +#if __has_cpp_attribute(nodiscard) +#define IMMER_NODISCARD [[nodiscard]] +#endif +#else +#if _MSVC_LANG >= 201703L +#define IMMER_NODISCARD [[nodiscard]] +#endif +#endif + +#ifdef __has_feature +#if !__has_feature(cxx_exceptions) +#define IMMER_NO_EXCEPTIONS +#endif +#endif + +#ifdef IMMER_NO_EXCEPTIONS +#define IMMER_TRY if (true) +#define IMMER_CATCH(expr) else +#define IMMER_THROW(expr) \ + do { \ + assert(!#expr); \ + std::terminate(); \ + } while (false) +#define IMMER_RETHROW +#else +#define IMMER_TRY try +#define IMMER_CATCH(expr) catch (expr) +#define IMMER_THROW(expr) throw expr +#define IMMER_RETHROW throw +#endif + +#ifndef IMMER_NODISCARD +#define IMMER_NODISCARD +#endif + +#ifndef IMMER_TAGGED_NODE +#ifdef NDEBUG +#define IMMER_TAGGED_NODE 0 +#else +#define IMMER_TAGGED_NODE 1 +#endif +#endif + +#if IMMER_TAGGED_NODE +#define IMMER_ASSERT_TAGGED(assertion) assert(assertion) +#else +#define IMMER_ASSERT_TAGGED(assertion) +#endif + #ifndef IMMER_DEBUG_TRACES #define IMMER_DEBUG_TRACES 0 #endif @@ -16,6 +70,10 @@ #define IMMER_DEBUG_PRINT 0 #endif +#ifndef IMMER_DEBUG_STATS +#define IMMER_DEBUG_STATS 0 +#endif + #ifndef IMMER_DEBUG_DEEP_CHECK #define IMMER_DEBUG_DEEP_CHECK 0 #endif @@ -25,42 +83,47 @@ #include #endif +#if IMMER_DEBUG_STATS +#include +#endif + #if IMMER_DEBUG_TRACES #define IMMER_TRACE(...) std::cout << __VA_ARGS__ << std::endl #else #define IMMER_TRACE(...) #endif -#define IMMER_TRACE_F(...) \ +#define IMMER_TRACE_F(...) \ IMMER_TRACE(__FILE__ << ":" << __LINE__ << ": " << __VA_ARGS__) -#define IMMER_TRACE_E(expr) \ - IMMER_TRACE(" " << #expr << " = " << (expr)) +#define IMMER_TRACE_E(expr) IMMER_TRACE(" " << #expr << " = " << (expr)) #if defined(_MSC_VER) -#define IMMER_UNREACHABLE __assume(false) -#define IMMER_LIKELY(cond) cond +#define IMMER_UNREACHABLE __assume(false) +#define IMMER_LIKELY(cond) cond #define IMMER_UNLIKELY(cond) cond -#define IMMER_FORCEINLINE __forceinline +#define IMMER_FORCEINLINE __forceinline #define IMMER_PREFETCH(p) #else -#define IMMER_UNREACHABLE __builtin_unreachable() -#define IMMER_LIKELY(cond) __builtin_expect(!!(cond), 1) +#define IMMER_UNREACHABLE __builtin_unreachable() +#define IMMER_LIKELY(cond) __builtin_expect(!!(cond), 1) #define IMMER_UNLIKELY(cond) __builtin_expect(!!(cond), 0) -#define IMMER_FORCEINLINE inline __attribute__ ((always_inline)) +#define IMMER_FORCEINLINE inline __attribute__((always_inline)) #define IMMER_PREFETCH(p) // #define IMMER_PREFETCH(p) __builtin_prefetch(p) #endif #define IMMER_DESCENT_DEEP 0 +#ifndef IMMER_ENABLE_DEBUG_SIZE_HEAP #ifdef NDEBUG #define IMMER_ENABLE_DEBUG_SIZE_HEAP 0 #else #define IMMER_ENABLE_DEBUG_SIZE_HEAP 1 #endif +#endif namespace immer { -const auto default_bits = 5; +const auto default_bits = 5; const auto default_free_list_size = 1 << 10; } // namespace immer diff --git a/src/immer/detail/arrays/no_capacity.hpp b/src/immer/detail/arrays/no_capacity.hpp index 33135d0cc3..a24f6da249 100644 --- a/src/immer/detail/arrays/no_capacity.hpp +++ b/src/immer/detail/arrays/no_capacity.hpp @@ -9,8 +9,13 @@ #pragma once #include +#include #include +#include +#include +#include + namespace immer { namespace detail { namespace arrays { @@ -18,16 +23,16 @@ namespace arrays { template struct no_capacity { - using node_t = node; - using edit_t = typename MemoryPolicy::transience_t::edit; - using size_t = std::size_t; + using node_t = node; + using edit_t = typename MemoryPolicy::transience_t::edit; + using size_t = std::size_t; node_t* ptr; - size_t size; + size_t size; static const no_capacity& empty() { - static const no_capacity empty_ { + static const no_capacity empty_{ node_t::make_n(0), 0, }; @@ -35,7 +40,8 @@ struct no_capacity } no_capacity(node_t* p, size_t s) - : ptr{p}, size{s} + : ptr{p} + , size{s} {} no_capacity(const no_capacity& other) @@ -70,10 +76,7 @@ struct no_capacity swap(x.size, y.size); } - ~no_capacity() - { - dec(); - } + ~no_capacity() { dec(); } void inc() { @@ -91,22 +94,33 @@ struct no_capacity T* data() { return ptr->data(); } const T* data() const { return ptr->data(); } - template - && compatible_sentinel_v, bool> = true> + T* data_mut(edit_t e) + { + if (!ptr->can_mutate(e)) + ptr = node_t::copy_e(e, size, ptr, size); + return data(); + } + + template && + compatible_sentinel_v, + bool> = true> static no_capacity from_range(Iter first, Sent last) { auto count = static_cast(distance(first, last)); - return { - node_t::copy_n(count, first, last), - count, - }; + if (count == 0) + return empty(); + else + return { + node_t::copy_n(count, first, last), + count, + }; } static no_capacity from_fill(size_t n, T v) { - return { node_t::fill_n(n, v), n }; + return {node_t::fill_n(n, v), n}; } template @@ -128,46 +142,45 @@ struct no_capacity return std::forward(fn)(data(), data() + size); } - const T& get(std::size_t index) const - { - return data()[index]; - } + const T& get(std::size_t index) const { return data()[index]; } const T& get_check(std::size_t index) const { if (index >= size) - throw std::out_of_range{"out of range"}; + IMMER_THROW(std::out_of_range{"out of range"}); return data()[index]; } bool equals(const no_capacity& other) const { return ptr == other.ptr || - (size == other.size && - std::equal(data(), data() + size, other.data())); + (size == other.size && + std::equal(data(), data() + size, other.data())); } no_capacity push_back(T value) const { auto p = node_t::copy_n(size + 1, ptr, size); - try { + IMMER_TRY { new (p->data() + size) T{std::move(value)}; - return { p, size + 1 }; - } catch (...) { + return {p, size + 1}; + } + IMMER_CATCH (...) { node_t::delete_n(p, size, size + 1); - throw; + IMMER_RETHROW; } } no_capacity assoc(std::size_t idx, T value) const { auto p = node_t::copy_n(size, ptr, size); - try { + IMMER_TRY { p->data()[idx] = std::move(value); - return { p, size }; - } catch (...) { + return {p, size}; + } + IMMER_CATCH (...) { node_t::delete_n(p, size, size); - throw; + IMMER_RETHROW; } } @@ -175,20 +188,21 @@ struct no_capacity no_capacity update(std::size_t idx, Fn&& op) const { auto p = node_t::copy_n(size, ptr, size); - try { + IMMER_TRY { auto& elem = p->data()[idx]; - elem = std::forward(op)(std::move(elem)); - return { p, size }; - } catch (...) { + elem = std::forward(op)(std::move(elem)); + return {p, size}; + } + IMMER_CATCH (...) { node_t::delete_n(p, size, size); - throw; + IMMER_RETHROW; } } no_capacity take(std::size_t sz) const { auto p = node_t::copy_n(sz, ptr, sz); - return { p, sz }; + return {p, sz}; } }; diff --git a/src/immer/detail/arrays/node.hpp b/src/immer/detail/arrays/node.hpp index 7883d167f2..ac521e0335 100644 --- a/src/immer/detail/arrays/node.hpp +++ b/src/immer/detail/arrays/node.hpp @@ -8,10 +8,12 @@ #pragma once -#include -#include +#include #include +#include +#include +#include #include namespace immer { @@ -21,54 +23,48 @@ namespace arrays { template struct node { - using memory = MemoryPolicy; - using heap = typename MemoryPolicy::heap::type; - using transience = typename memory::transience_t; - using refs_t = typename memory::refcount; - using ownee_t = typename transience::ownee; - using node_t = node; - using edit_t = typename transience::edit; + using memory = MemoryPolicy; + using heap = typename MemoryPolicy::heap::type; + using transience = typename memory::transience_t; + using refs_t = typename memory::refcount; + using ownee_t = typename transience::ownee; + using node_t = node; + using edit_t = typename transience::edit; struct data_t { aligned_storage_for buffer; }; - using impl_t = combine_standard_layout_t; + using impl_t = combine_standard_layout_t; impl_t impl; constexpr static std::size_t sizeof_n(size_t count) { - return immer_offsetof(impl_t, d.buffer) + sizeof(T) * count; + return std::max(immer_offsetof(impl_t, d.buffer) + sizeof(T) * count, + sizeof(node)); } - refs_t& refs() const - { - return auto_const_cast(get(impl)); - } + refs_t& refs() const { return auto_const_cast(get(impl)); } const ownee_t& ownee() const { return get(impl); } - ownee_t& ownee() { return get(impl); } + ownee_t& ownee() { return get(impl); } const T* data() const { return reinterpret_cast(&impl.d.buffer); } - T* data() { return reinterpret_cast(&impl.d.buffer); } + T* data() { return reinterpret_cast(&impl.d.buffer); } bool can_mutate(edit_t e) const { - return refs().unique() - || ownee().can_mutate(e); + return refs().unique() || ownee().can_mutate(e); } static void delete_n(node_t* p, size_t sz, size_t cap) { - destroy_n(p->data(), sz); + detail::destroy_n(p->data(), sz); heap::deallocate(sizeof_n(cap), p); } - static node_t* make_n(size_t n) { return new (heap::allocate(sizeof_n(n))) node_t{}; @@ -76,7 +72,7 @@ struct node static node_t* make_e(edit_t e, size_t n) { - auto p = make_n(n); + auto p = make_n(n); p->ownee() = e; return p; } @@ -84,27 +80,30 @@ struct node static node_t* fill_n(size_t n, T v) { auto p = make_n(n); - try { + IMMER_TRY { std::uninitialized_fill_n(p->data(), n, v); return p; - } catch (...) { + } + IMMER_CATCH (...) { heap::deallocate(sizeof_n(n), p); - throw; + IMMER_RETHROW; } } - template , bool> = true> + template , + bool> = true> static node_t* copy_n(size_t n, Iter first, Sent last) { auto p = make_n(n); - try { - uninitialized_copy(first, last, p->data()); + IMMER_TRY { + detail::uninitialized_copy(first, last, p->data()); return p; - } catch (...) { + } + IMMER_CATCH (...) { heap::deallocate(sizeof_n(n), p); - throw; + IMMER_RETHROW; } } @@ -116,7 +115,7 @@ struct node template static node_t* copy_e(edit_t e, size_t n, Iter first, Iter last) { - auto p = copy_n(n, first, last); + auto p = copy_n(n, first, last); p->ownee() = e; return p; } diff --git a/src/immer/detail/arrays/with_capacity.hpp b/src/immer/detail/arrays/with_capacity.hpp index d408b38aa3..c80f706821 100644 --- a/src/immer/detail/arrays/with_capacity.hpp +++ b/src/immer/detail/arrays/with_capacity.hpp @@ -8,8 +8,13 @@ #pragma once +#include #include +#include +#include +#include + namespace immer { namespace detail { namespace arrays { @@ -19,26 +24,24 @@ struct with_capacity { using no_capacity_t = no_capacity; - using node_t = node; - using edit_t = typename MemoryPolicy::transience_t::edit; - using size_t = std::size_t; + using node_t = node; + using edit_t = typename MemoryPolicy::transience_t::edit; + using size_t = std::size_t; node_t* ptr; - size_t size; - size_t capacity; + size_t size; + size_t capacity; static const with_capacity& empty() { - static const with_capacity empty_ { - node_t::make_n(1), - 0, - 1 - }; + static const with_capacity empty_{node_t::make_n(1), 0, 1}; return empty_; } with_capacity(node_t* p, size_t s, size_t c) - : ptr{p}, size{s}, capacity{c} + : ptr{p} + , size{s} + , capacity{c} {} with_capacity(const with_capacity& other) @@ -80,10 +83,7 @@ struct with_capacity swap(x.capacity, y.capacity); } - ~with_capacity() - { - dec(); - } + ~with_capacity() { dec(); } void inc() { @@ -99,30 +99,40 @@ struct with_capacity } const T* data() const { return ptr->data(); } - T* data() { return ptr->data(); } + T* data() { return ptr->data(); } + + T* data_mut(edit_t e) + { + if (!ptr->can_mutate(e)) { + auto p = node_t::copy_e(e, capacity, ptr, size); + dec(); + ptr = p; + } + return data(); + } operator no_capacity_t() const { if (size == capacity) { ptr->refs().inc(); - return { ptr, size }; + return {ptr, size}; } else { - return { node_t::copy_n(size, ptr, size), size }; + return {node_t::copy_n(size, ptr, size), size}; } } - template - && compatible_sentinel_v, bool> = true> + template && + compatible_sentinel_v, + bool> = true> static with_capacity from_range(Iter first, Sent last) { auto count = static_cast(distance(first, last)); - return { - node_t::copy_n(count, first, last), - count, - count - }; + if (count == 0) + return empty(); + else + return {node_t::copy_n(count, first, last), count, count}; } template @@ -134,7 +144,7 @@ struct with_capacity static with_capacity from_fill(size_t n, T v) { - return { node_t::fill_n(n, v), n, n }; + return {node_t::fill_n(n, v), n, n}; } template @@ -149,51 +159,50 @@ struct with_capacity return std::forward(fn)(data(), data() + size); } - const T& get(std::size_t index) const - { - return data()[index]; - } + const T& get(std::size_t index) const { return data()[index]; } const T& get_check(std::size_t index) const { if (index >= size) - throw std::out_of_range{"out of range"}; + IMMER_THROW(std::out_of_range{"out of range"}); return data()[index]; } bool equals(const with_capacity& other) const { return ptr == other.ptr || - (size == other.size && - std::equal(data(), data() + size, other.data())); + (size == other.size && + std::equal(data(), data() + size, other.data())); } static size_t recommend_up(size_t sz, size_t cap) { auto max = std::numeric_limits::max(); - return - sz <= cap ? cap : - cap >= max / 2 ? max - /* otherwise */ : std::max(2 * cap, sz); + return sz <= cap ? cap + : cap >= max / 2 ? max + /* otherwise */ + : std::max(2 * cap, sz); } static size_t recommend_down(size_t sz, size_t cap) { - return sz == 0 ? 1 : - sz < cap / 2 ? sz * 2 : - /* otherwise */ cap; + return sz == 0 ? 1 + : sz < cap / 2 ? sz * 2 + : + /* otherwise */ cap; } with_capacity push_back(T value) const { auto cap = recommend_up(size + 1, capacity); - auto p = node_t::copy_n(cap, ptr, size); - try { + auto p = node_t::copy_n(cap, ptr, size); + IMMER_TRY { new (p->data() + size) T{std::move(value)}; - return { p, size + 1, cap }; - } catch (...) { + return {p, size + 1, cap}; + } + IMMER_CATCH (...) { node_t::delete_n(p, size, cap); - throw; + IMMER_RETHROW; } } @@ -204,13 +213,14 @@ struct with_capacity ++size; } else { auto cap = recommend_up(size + 1, capacity); - auto p = node_t::copy_e(e, cap, ptr, size); - try { + auto p = node_t::copy_e(e, cap, ptr, size); + IMMER_TRY { new (p->data() + size) T{std::move(value)}; - *this = { p, size + 1, cap }; - } catch (...) { + *this = {p, size + 1, cap}; + } + IMMER_CATCH (...) { node_t::delete_n(p, size, cap); - throw; + IMMER_RETHROW; } } } @@ -218,12 +228,13 @@ struct with_capacity with_capacity assoc(std::size_t idx, T value) const { auto p = node_t::copy_n(capacity, ptr, size); - try { + IMMER_TRY { p->data()[idx] = std::move(value); - return { p, size, capacity }; - } catch (...) { + return {p, size, capacity}; + } + IMMER_CATCH (...) { node_t::delete_n(p, size, capacity); - throw; + IMMER_RETHROW; } } @@ -233,12 +244,13 @@ struct with_capacity data()[idx] = std::move(value); } else { auto p = node_t::copy_n(capacity, ptr, size); - try { + IMMER_TRY { p->data()[idx] = std::move(value); - *this = { p, size, capacity }; - } catch (...) { + *this = {p, size, capacity}; + } + IMMER_CATCH (...) { node_t::delete_n(p, size, capacity); - throw; + IMMER_RETHROW; } } } @@ -247,13 +259,14 @@ struct with_capacity with_capacity update(std::size_t idx, Fn&& op) const { auto p = node_t::copy_n(capacity, ptr, size); - try { + IMMER_TRY { auto& elem = p->data()[idx]; - elem = std::forward(op)(std::move(elem)); - return { p, size, capacity }; - } catch (...) { + elem = std::forward(op)(std::move(elem)); + return {p, size, capacity}; + } + IMMER_CATCH (...) { node_t::delete_n(p, size, capacity); - throw; + IMMER_RETHROW; } } @@ -262,36 +275,39 @@ struct with_capacity { if (ptr->can_mutate(e)) { auto& elem = data()[idx]; - elem = std::forward(op)(std::move(elem)); + elem = std::forward(op)(std::move(elem)); } else { auto p = node_t::copy_e(e, capacity, ptr, size); - try { + IMMER_TRY { auto& elem = p->data()[idx]; - elem = std::forward(op)(std::move(elem)); - *this = { p, size, capacity }; - } catch (...) { + elem = std::forward(op)(std::move(elem)); + *this = {p, size, capacity}; + } + IMMER_CATCH (...) { node_t::delete_n(p, size, capacity); - throw; + IMMER_RETHROW; } } } with_capacity take(std::size_t sz) const { + assert(sz <= size); auto cap = recommend_down(sz, capacity); - auto p = node_t::copy_n(cap, ptr, sz); - return { p, sz, cap }; + auto p = node_t::copy_n(cap, ptr, sz); + return {p, sz, cap}; } void take_mut(edit_t e, std::size_t sz) { + assert(sz <= size); if (ptr->can_mutate(e)) { - destroy_n(data() + size, size - sz); + detail::destroy_n(data() + size, size - sz); size = sz; } else { auto cap = recommend_down(sz, capacity); - auto p = node_t::copy_e(e, cap, ptr, sz); - *this = { p, sz, cap }; + auto p = node_t::copy_e(e, cap, ptr, sz); + *this = {p, sz, cap}; } } }; diff --git a/src/immer/detail/combine_standard_layout.hpp b/src/immer/detail/combine_standard_layout.hpp index be8e698acc..68de2a8a8c 100644 --- a/src/immer/detail/combine_standard_layout.hpp +++ b/src/immer/detail/combine_standard_layout.hpp @@ -8,11 +8,12 @@ #pragma once +#include #include -#if __GNUC__ == 7 || __GNUC_MINOR__ == 1 +#if defined(__GNUC__) && __GNUC__ == 7 && __GNUC_MINOR__ == 1 #define IMMER_BROKEN_STANDARD_LAYOUT_DETECTION 1 -#define immer_offsetof(st, m) ((std::size_t) &(((st*)0)->m)) +#define immer_offsetof(st, m) ((std::size_t) & (((st*) 0)->m)) #else #define IMMER_BROKEN_STANDARD_LAYOUT_DETECTION 0 #define immer_offsetof offsetof @@ -48,7 +49,8 @@ using combine_standard_layout_t = typename combine_standard_layout::type; namespace csl { template -struct type_t {}; +struct type_t +{}; template U& get(T& x); @@ -56,17 +58,25 @@ U& get(T& x); template const U& get(const T& x); -template +template struct inherit { - struct type : T, Next + struct type + : T + , Next { using Next::get_; template - friend decltype(auto) get(type& x) { return x.get_(type_t{}); } + friend decltype(auto) get(type& x) + { + return x.get_(type_t{}); + } template - friend decltype(auto) get(const type& x) { return x.get_(type_t{}); } + friend decltype(auto) get(const type& x) + { + return x.get_(type_t{}); + } T& get_(type_t) { return *this; } const T& get_(type_t) const { return *this; } @@ -79,16 +89,22 @@ struct inherit struct type : T { template - friend decltype(auto) get(type& x) { return x.get_(type_t{}); } + friend decltype(auto) get(type& x) + { + return x.get_(type_t{}); + } template - friend decltype(auto) get(const type& x) { return x.get_(type_t{}); } + friend decltype(auto) get(const type& x) + { + return x.get_(type_t{}); + } T& get_(type_t) { return *this; } const T& get_(type_t) const { return *this; } }; }; -template +template struct member { struct type : Next @@ -98,9 +114,15 @@ struct member using Next::get_; template - friend decltype(auto) get(type& x) { return x.get_(type_t{}); } + friend decltype(auto) get(type& x) + { + return x.get_(type_t{}); + } template - friend decltype(auto) get(const type& x) { return x.get_(type_t{}); } + friend decltype(auto) get(const type& x) + { + return x.get_(type_t{}); + } T& get_(type_t) { return d; } const T& get_(type_t) const { return d; } @@ -115,9 +137,15 @@ struct member T d; template - friend decltype(auto) get(type& x) { return x.get_(type_t{}); } + friend decltype(auto) get(type& x) + { + return x.get_(type_t{}); + } template - friend decltype(auto) get(const type& x) { return x.get_(type_t{}); } + friend decltype(auto) get(const type& x) + { + return x.get_(type_t{}); + } T& get_(type_t) { return d; } const T& get_(type_t) const { return d; } @@ -133,17 +161,29 @@ struct member_two T d; template - friend decltype(auto) get(type& x) { return x.get_(type_t{}); } + friend decltype(auto) get(type& x) + { + return x.get_(type_t{}); + } template - friend decltype(auto) get(const type& x) { return x.get_(type_t{}); } + friend decltype(auto) get(const type& x) + { + return x.get_(type_t{}); + } T& get_(type_t) { return d; } const T& get_(type_t) const { return d; } template - auto get_(type_t t) -> decltype(auto) { return n.get_(t); } + auto get_(type_t t) -> decltype(auto) + { + return n.get_(t); + } template - auto get_(type_t t) const -> decltype(auto) { return n.get_(t); } + auto get_(type_t t) const -> decltype(auto) + { + return n.get_(t); + } }; }; @@ -155,10 +195,9 @@ struct combine_standard_layout_aux { static_assert(std::is_standard_layout::value, ""); - using type = typename std::conditional_t< - std::is_empty::value, - csl::inherit, - csl::member>::type; + using type = typename std::conditional_t::value, + csl::inherit, + csl::member>::type; }; template @@ -173,10 +212,11 @@ struct combine_standard_layout_aux static constexpr auto empty_next = std::is_empty::value; using type = typename std::conditional_t< - empty_this, inherit, - std::conditional_t< - empty_next, member, - member_two>>::type; + empty_this, + inherit, + std::conditional_t, + member_two>>::type; }; } // namespace csl diff --git a/src/immer/detail/hamts/bits.hpp b/src/immer/detail/hamts/bits.hpp index 92c7e45cf2..0b4b923784 100644 --- a/src/immer/detail/hamts/bits.hpp +++ b/src/immer/detail/hamts/bits.hpp @@ -8,6 +8,7 @@ #pragma once +#include #include #if defined(_MSC_VER) @@ -18,18 +19,17 @@ namespace immer { namespace detail { namespace hamts { -using size_t = std::size_t; -using hash_t = std::size_t; -using bits_t = std::uint32_t; -using count_t = std::uint32_t; -using shift_t = std::uint32_t; +using size_t = std::size_t; +using hash_t = std::size_t; +using bits_t = std::uint32_t; +using count_t = std::uint32_t; +using shift_t = std::uint32_t; template struct get_bitmap_type { static_assert(B < 6u, "B > 6 is not supported."); - - using type = std::uint32_t; + using type = std::uint8_t; }; template <> @@ -38,17 +38,29 @@ struct get_bitmap_type<6u> using type = std::uint64_t; }; -template +template <> +struct get_bitmap_type<5u> +{ + using type = std::uint32_t; +}; + +template <> +struct get_bitmap_type<4u> +{ + using type = std::uint16_t; +}; + +template constexpr T branches = T{1u} << B; -template +template constexpr T mask = branches - 1u; -template +template constexpr T max_depth = (sizeof(hash_t) * 8u + B - 1u) / B; -template -constexpr T max_shift = max_depth * B; +template +constexpr T max_shift = max_depth* B; #define IMMER_HAS_BUILTIN_POPCOUNT 1 @@ -67,17 +79,18 @@ inline auto popcount_fallback(std::uint64_t x) { x = x - ((x >> 1) & 0x5555555555555555u); x = (x & 0x3333333333333333u) + ((x >> 2u) & 0x3333333333333333u); - return (((x + (x >> 4)) & 0x0F0F0F0F0F0F0F0Fu) * 0x0101010101010101u) >> 56u; + return (((x + (x >> 4)) & 0x0F0F0F0F0F0F0F0Fu) * 0x0101010101010101u) >> + 56u; } inline count_t popcount(std::uint32_t x) { #if IMMER_HAS_BUILTIN_POPCOUNT -# if defined(_MSC_VER) - return __popcnt(x); -# else +#if defined(_MSC_VER) + return __popcnt(x); +#else return __builtin_popcount(x); -# endif +#endif #else return popcount_fallback(x); #endif @@ -86,16 +99,77 @@ inline count_t popcount(std::uint32_t x) inline count_t popcount(std::uint64_t x) { #if IMMER_HAS_BUILTIN_POPCOUNT -# if defined(_MSC_VER) - return __popcnt64(x); -# else +#if defined(_MSC_VER) +#if defined(_WIN64) + return static_cast(__popcnt64(x)); +#else + // TODO: benchmark against popcount_fallback(std::uint64_t x) + return popcount(static_cast(x >> 32)) + + popcount(static_cast(x)); +#endif +#else return __builtin_popcountll(x); -# endif +#endif #else return popcount_fallback(x); #endif } +inline count_t popcount(std::uint16_t x) +{ + return popcount(static_cast(x)); +} + +inline count_t popcount(std::uint8_t x) +{ + return popcount(static_cast(x)); +} + +template +class set_bits_range +{ + bitmap_t bitmap; + + class set_bits_iterator + { + bitmap_t bitmap; + + inline static bitmap_t clearlsbit(bitmap_t bitmap) + { + return bitmap & (bitmap - 1); + } + + inline static bitmap_t lsbit(bitmap_t bitmap) + { + return bitmap ^ clearlsbit(bitmap); + } + + public: + set_bits_iterator(bitmap_t bitmap) + : bitmap(bitmap){}; + + set_bits_iterator operator++() + { + bitmap = clearlsbit(bitmap); + return *this; + } + + bool operator!=(set_bits_iterator const& other) const + { + return bitmap != other.bitmap; + } + + bitmap_t operator*() const { return lsbit(bitmap); } + }; + +public: + set_bits_range(bitmap_t bitmap) + : bitmap(bitmap) + {} + set_bits_iterator begin() const { return set_bits_iterator(bitmap); } + set_bits_iterator end() const { return set_bits_iterator(0); } +}; + } // namespace hamts } // namespace detail } // namespace immer diff --git a/src/immer/detail/hamts/champ.hpp b/src/immer/detail/hamts/champ.hpp index d3687ffbee..64a8cb8248 100644 --- a/src/immer/detail/hamts/champ.hpp +++ b/src/immer/detail/hamts/champ.hpp @@ -17,6 +17,107 @@ namespace immer { namespace detail { namespace hamts { +#if IMMER_DEBUG_STATS +struct champ_debug_stats +{ + std::size_t bits = {}; + std::size_t value_size = {}; + std::size_t child_size = sizeof(void*); + + std::size_t inner_node_count = {}; + std::size_t inner_node_w_value_count = {}; + std::size_t inner_node_w_child_count = {}; + std::size_t collision_node_count = {}; + + std::size_t child_count = {}; + std::size_t value_count = {}; + std::size_t collision_count = {}; + + friend champ_debug_stats operator+(champ_debug_stats a, champ_debug_stats b) + { + if (a.bits != b.bits || a.value_size != b.value_size || + a.child_size != b.child_size) + throw std::runtime_error{"accumulating incompatible stats"}; + return { + a.bits, + a.value_size, + a.child_size, + a.inner_node_count + b.inner_node_count, + a.inner_node_w_value_count + b.inner_node_w_value_count, + a.inner_node_w_child_count + b.inner_node_w_child_count, + a.collision_node_count + b.collision_node_count, + a.child_count + b.child_count, + a.value_count + b.value_count, + a.collision_count + b.collision_count, + }; + } + + struct summary + { + double collision_ratio; + + double utilization; + double child_utilization; + double value_utilization; + + double dense_utilization; + double dense_value_utilization; + double dense_child_utilization; + + friend std::ostream& operator<<(std::ostream& os, const summary& s) + { + os << "---\n"; + os << "collisions\n" + << " ratio = " << s.collision_ratio << " %\n"; + os << "utilization\n" + << " total = " << s.utilization << " %\n" + << " children = " << s.child_utilization << " %\n" + << " values = " << s.value_utilization << " %\n"; + os << "utilization (dense)\n" + << " total = " << s.dense_utilization << " %\n" + << " children = " << s.dense_child_utilization << " %\n" + << " values = " << s.dense_value_utilization << " %\n"; + return os; + } + }; + + summary get_summary() const + { + auto m = std::size_t{1} << bits; + + auto collision_ratio = 100. * collision_count / value_count; + + auto capacity = m * inner_node_count; + auto child_utilization = 100. * child_count / capacity; + auto value_utilization = 100. * value_count / capacity; + auto utilization = + 100. * (value_count * value_size + child_count * child_size) / + (capacity * value_size + capacity * child_size); + + auto value_capacity = m * inner_node_w_value_count; + auto child_capacity = m * inner_node_w_child_count; + auto dense_child_utilization = + child_capacity == 0 ? 100. : 100. * child_count / child_capacity; + auto dense_value_utilization = + value_capacity == 0 ? 100. : 100. * value_count / value_capacity; + auto dense_utilization = + value_capacity + child_capacity == 0 + ? 100. + : 100. * (value_count * value_size + child_count * child_size) / + (value_capacity * value_size + + child_capacity * child_size); + + return {collision_ratio, + utilization, + child_utilization, + value_utilization, + dense_utilization, + dense_value_utilization, + dense_child_utilization}; + } +}; +#endif + template ; + using node_t = node; + using edit_t = typename MemoryPolicy::transience_t::edit; + using owner_t = typename MemoryPolicy::transience_t::owner; using bitmap_t = typename get_bitmap_type::type; static_assert(branches <= sizeof(bitmap_t) * 8, ""); node_t* root; - size_t size; + size_t size; - static const champ& empty() + static node_t* empty() { - static const champ empty_ { - node_t::make_inner_n(0), - 0, - }; - return empty_; + static const auto node = node_t::make_inner_n(0); + return node->inc(); } - champ(node_t* r, size_t sz) - : root{r}, size{sz} - { - } + champ(node_t* r, size_t sz = 0) + : root{r} + , size{sz} + {} champ(const champ& other) : champ{other.root, other.size} @@ -80,20 +180,125 @@ struct champ swap(x.size, y.size); } - ~champ() + ~champ() { dec(); } + + void inc() const { root->inc(); } + + void dec() const { - dec(); + if (root->dec()) + node_t::delete_deep(root, 0); } - void inc() const + std::size_t do_check_champ(node_t* node, + count_t depth, + size_t path_hash, + size_t hash_mask) const { - root->inc(); + auto result = std::size_t{}; + if (depth < max_depth) { + auto nodemap = node->nodemap(); + if (nodemap) { + auto fst = node->children(); + for (auto idx = std::size_t{}; idx < branches; ++idx) { + if (nodemap & (1 << idx)) { + auto child = *fst++; + result += + do_check_champ(child, + depth + 1, + path_hash | (idx << (B * depth)), + (hash_mask << B) | mask); + } + } + } + auto datamap = node->datamap(); + if (datamap) { + auto fst = node->values(); + for (auto idx = std::size_t{}; idx < branches; ++idx) { + if (datamap & (1 << idx)) { + auto hash = Hash{}(*fst++); + auto check = (hash & hash_mask) == + (path_hash | (idx << (B * depth))); + // assert(check); + result += !!check; + } + } + } + } else { + auto fst = node->collisions(); + auto lst = fst + node->collision_count(); + for (; fst != lst; ++fst) { + auto hash = Hash{}(*fst); + auto check = hash == path_hash; + // assert(check); + result += !!check; + } + } + return result; } - void dec() const + // Checks that the hashes of the values correspond with what has actually + // been inserted. If it doesn't it can mean that corruption has happened + // due some value being moved out of the champ when it should have not. + bool check_champ() const { - if (root->dec()) - node_t::delete_deep(root, 0); + auto r = do_check_champ(root, 0, 0, mask); + // assert(r == size); + return r == size; + } + +#if IMMER_DEBUG_STATS + void do_get_debug_stats(champ_debug_stats& stats, + node_t* node, + count_t depth) const + { + if (depth < max_depth) { + ++stats.inner_node_count; + stats.inner_node_w_value_count += node->data_count() > 0; + stats.inner_node_w_child_count += node->children_count() > 0; + stats.value_count += node->data_count(); + stats.child_count += node->children_count(); + auto nodemap = node->nodemap(); + if (nodemap) { + auto fst = node->children(); + auto lst = fst + node->children_count(); + for (; fst != lst; ++fst) + do_get_debug_stats(stats, *fst, depth + 1); + } + } else { + ++stats.collision_node_count; + stats.collision_count += node->collision_count(); + } + } + + champ_debug_stats get_debug_stats() const + { + auto stats = champ_debug_stats{B, sizeof(T)}; + do_get_debug_stats(stats, root, 0); + return stats; + } +#endif + + template + static auto from_initializer_list(std::initializer_list values) + { + auto e = owner_t{}; + auto result = champ{empty()}; + for (auto&& v : values) + result.add_mut(e, v); + return result; + } + + template , bool> = true> + static auto from_range(Iter first, Sent last) + { + auto e = owner_t{}; + auto result = champ{empty()}; + for (; first != last; ++first) + result.add_mut(e, *first); + return result; } template @@ -103,21 +308,221 @@ struct champ } template - void for_each_chunk_traversal(node_t* node, count_t depth, Fn&& fn) const + void + for_each_chunk_traversal(const node_t* node, count_t depth, Fn&& fn) const { if (depth < max_depth) { auto datamap = node->datamap(); if (datamap) - fn(node->values(), node->values() + popcount(datamap)); + fn(node->values(), node->values() + node->data_count()); auto nodemap = node->nodemap(); if (nodemap) { auto fst = node->children(); - auto lst = fst + popcount(nodemap); + auto lst = fst + node->children_count(); for (; fst != lst; ++fst) for_each_chunk_traversal(*fst, depth + 1, fn); } } else { - fn(node->collisions(), node->collisions() + node->collision_count()); + fn(node->collisions(), + node->collisions() + node->collision_count()); + } + } + + template + void diff(const champ& new_champ, Differ&& differ) const + { + diff(root, new_champ.root, 0, std::forward(differ)); + } + + template + void diff(const node_t* old_node, + const node_t* new_node, + count_t depth, + Differ&& differ) const + { + if (old_node == new_node) + return; + if (depth < max_depth) { + auto old_nodemap = old_node->nodemap(); + auto new_nodemap = new_node->nodemap(); + auto old_datamap = old_node->datamap(); + auto new_datamap = new_node->datamap(); + auto old_bits = old_nodemap | old_datamap; + auto new_bits = new_nodemap | new_datamap; + auto changes = old_bits ^ new_bits; + + // added bits + for (auto bit : set_bits_range(new_bits & changes)) { + if (new_nodemap & bit) { + auto offset = new_node->children_count(bit); + auto child = new_node->children()[offset]; + for_each_chunk_traversal( + child, + depth + 1, + [&](auto const& begin, auto const& end) { + for (auto it = begin; it != end; it++) + differ.added(*it); + }); + } else if (new_datamap & bit) { + auto offset = new_node->data_count(bit); + auto const& value = new_node->values()[offset]; + differ.added(value); + } + } + + // removed bits + for (auto bit : set_bits_range(old_bits & changes)) { + if (old_nodemap & bit) { + auto offset = old_node->children_count(bit); + auto child = old_node->children()[offset]; + for_each_chunk_traversal( + child, + depth + 1, + [&](auto const& begin, auto const& end) { + for (auto it = begin; it != end; it++) + differ.removed(*it); + }); + } else if (old_datamap & bit) { + auto offset = old_node->data_count(bit); + auto const& value = old_node->values()[offset]; + differ.removed(value); + } + } + + // bits in both nodes + for (auto bit : set_bits_range(old_bits & new_bits)) { + if ((old_nodemap & bit) && (new_nodemap & bit)) { + auto old_offset = old_node->children_count(bit); + auto new_offset = new_node->children_count(bit); + auto old_child = old_node->children()[old_offset]; + auto new_child = new_node->children()[new_offset]; + diff(old_child, new_child, depth + 1, differ); + } else if ((old_datamap & bit) && (new_nodemap & bit)) { + diff_data_node( + old_node, new_node, bit, depth, differ); + } else if ((old_nodemap & bit) && (new_datamap & bit)) { + diff_node_data( + old_node, new_node, bit, depth, differ); + } else if ((old_datamap & bit) && (new_datamap & bit)) { + diff_data_data(old_node, new_node, bit, differ); + } + } + } else { + diff_collisions(old_node, new_node, differ); + } + } + + template + void diff_data_node(const node_t* old_node, + const node_t* new_node, + bitmap_t bit, + count_t depth, + Differ&& differ) const + { + auto old_offset = old_node->data_count(bit); + auto const& old_value = old_node->values()[old_offset]; + auto new_offset = new_node->children_count(bit); + auto new_child = new_node->children()[new_offset]; + + bool found = false; + for_each_chunk_traversal( + new_child, depth + 1, [&](auto const& begin, auto const& end) { + for (auto it = begin; it != end; it++) { + if (Equal{}(old_value, *it)) { + if (!EqualValue{}(old_value, *it)) + differ.changed(old_value, *it); + found = true; + } else { + differ.added(*it); + } + } + }); + if (!found) + differ.removed(old_value); + } + + template + void diff_node_data(const node_t* old_node, + const node_t* const new_node, + bitmap_t bit, + count_t depth, + Differ&& differ) const + { + auto old_offset = old_node->children_count(bit); + auto old_child = old_node->children()[old_offset]; + auto new_offset = new_node->data_count(bit); + auto const& new_value = new_node->values()[new_offset]; + + bool found = false; + for_each_chunk_traversal( + old_child, depth + 1, [&](auto const& begin, auto const& end) { + for (auto it = begin; it != end; it++) { + if (Equal{}(*it, new_value)) { + if (!EqualValue{}(*it, new_value)) + differ.changed(*it, new_value); + found = true; + } else { + differ.removed(*it); + } + } + }); + if (!found) + differ.added(new_value); + } + + template + void diff_data_data(const node_t* old_node, + const node_t* new_node, + bitmap_t bit, + Differ&& differ) const + { + auto old_offset = old_node->data_count(bit); + auto new_offset = new_node->data_count(bit); + auto const& old_value = old_node->values()[old_offset]; + auto const& new_value = new_node->values()[new_offset]; + if (!Equal{}(old_value, new_value)) { + differ.removed(old_value); + differ.added(new_value); + } else { + if (!EqualValue{}(old_value, new_value)) + differ.changed(old_value, new_value); + } + } + + template + void diff_collisions(const node_t* old_node, + const node_t* new_node, + Differ&& differ) const + { + auto old_begin = old_node->collisions(); + auto old_end = old_node->collisions() + old_node->collision_count(); + auto new_begin = new_node->collisions(); + auto new_end = new_node->collisions() + new_node->collision_count(); + // search changes and removals + for (auto old_it = old_begin; old_it != old_end; old_it++) { + bool found = false; + for (auto new_it = new_begin; new_it != new_end; new_it++) { + if (Equal{}(*old_it, *new_it)) { + if (!EqualValue{}(*old_it, *new_it)) + differ.changed(*old_it, *new_it); + found = true; + break; + } + } + if (!found) + differ.removed(*old_it); + } + // search new entries + for (auto new_it = new_begin; new_it != new_end; new_it++) { + bool found = false; + for (auto old_it = old_begin; old_it != old_end; old_it++) { + if (Equal{}(*old_it, *new_it)) { + found = true; + break; + } + } + if (!found) + differ.added(*new_it); } } @@ -129,11 +534,11 @@ struct champ for (auto i = count_t{}; i < max_depth; ++i) { auto bit = bitmap_t{1u} << (hash & mask); if (node->nodemap() & bit) { - auto offset = popcount(node->nodemap() & (bit - 1)); - node = node->children() [offset]; - hash = hash >> B; + auto offset = node->children_count(bit); + node = node->children()[offset]; + hash = hash >> B; } else if (node->datamap() & bit) { - auto offset = popcount(node->datamap() & (bit - 1)); + auto offset = node->data_count(bit); auto val = node->values() + offset; if (Equal{}(*val, k)) return Project{}(*val); @@ -151,9 +556,15 @@ struct champ return Default{}(); } - std::pair - do_add(node_t* node, T v, hash_t hash, shift_t shift) const + struct add_result + { + node_t* node; + bool added; + }; + + add_result do_add(node_t* node, T v, hash_t hash, shift_t shift) const { + assert(node); if (shift == max_shift) { auto fst = node->collisions(); auto lst = fst + node->collision_count(); @@ -161,159 +572,591 @@ struct champ if (Equal{}(*fst, v)) return { node_t::copy_collision_replace(node, fst, std::move(v)), - false - }; - return { - node_t::copy_collision_insert(node, std::move(v)), - true - }; + false}; + return {node_t::copy_collision_insert(node, std::move(v)), true}; } else { auto idx = (hash & (mask << shift)) >> shift; auto bit = bitmap_t{1u} << idx; if (node->nodemap() & bit) { - auto offset = popcount(node->nodemap() & (bit - 1)); - auto result = do_add(node->children() [offset], - std::move(v), hash, - shift + B); - try { - result.first = node_t::copy_inner_replace( - node, offset, result.first); + auto offset = node->children_count(bit); + assert(node->children()[offset]); + auto result = do_add( + node->children()[offset], std::move(v), hash, shift + B); + IMMER_TRY { + result.node = + node_t::copy_inner_replace(node, offset, result.node); return result; - } catch (...) { - node_t::delete_deep_shift(result.first, shift + B); - throw; + } + IMMER_CATCH (...) { + node_t::delete_deep_shift(result.node, shift + B); + IMMER_RETHROW; } } else if (node->datamap() & bit) { - auto offset = popcount(node->datamap() & (bit - 1)); + auto offset = node->data_count(bit); auto val = node->values() + offset; if (Equal{}(*val, v)) - return { - node_t::copy_inner_replace_value( - node, offset, std::move(v)), - false - }; + return {node_t::copy_inner_replace_value( + node, offset, std::move(v)), + false}; else { - auto child = node_t::make_merged(shift + B, - std::move(v), hash, - *val, Hash{}(*val)); - try { - return { - node_t::copy_inner_replace_merged( - node, bit, offset, child), - true - }; - } catch (...) { + auto child = node_t::make_merged( + shift + B, std::move(v), hash, *val, Hash{}(*val)); + IMMER_TRY { + return {node_t::copy_inner_replace_merged( + node, bit, offset, child), + true}; + } + IMMER_CATCH (...) { node_t::delete_deep_shift(child, shift + B); - throw; + IMMER_RETHROW; } } } else { return { node_t::copy_inner_insert_value(node, bit, std::move(v)), - true - }; + true}; } } } champ add(T v) const + { + auto hash = Hash{}(v); + auto res = do_add(root, std::move(v), hash, 0); + auto new_size = size + (res.added ? 1 : 0); + return {res.node, new_size}; + } + + struct add_mut_result + { + node_t* node; + bool added; + bool mutated; + }; + + add_mut_result + do_add_mut(edit_t e, node_t* node, T v, hash_t hash, shift_t shift) const + { + assert(node); + if (shift == max_shift) { + auto fst = node->collisions(); + auto lst = fst + node->collision_count(); + for (; fst != lst; ++fst) + if (Equal{}(*fst, v)) { + if (node->can_mutate(e)) { + *fst = std::move(v); + return {node, false, true}; + } else { + auto r = node_t::copy_collision_replace( + node, fst, std::move(v)); + return {node_t::owned(r, e), false, false}; + } + } + auto mutate = node->can_mutate(e); + auto r = mutate ? node_t::move_collision_insert(node, std::move(v)) + : node_t::copy_collision_insert(node, std::move(v)); + return {node_t::owned(r, e), true, mutate}; + } else { + auto idx = (hash & (mask << shift)) >> shift; + auto bit = bitmap_t{1u} << idx; + if (node->nodemap() & bit) { + auto offset = node->children_count(bit); + auto child = node->children()[offset]; + if (node->can_mutate(e)) { + auto result = + do_add_mut(e, child, std::move(v), hash, shift + B); + node->children()[offset] = result.node; + if (!result.mutated && child->dec()) + node_t::delete_deep_shift(child, shift + B); + return {node, result.added, true}; + } else { + assert(node->children()[offset]); + auto result = do_add(child, std::move(v), hash, shift + B); + IMMER_TRY { + result.node = node_t::copy_inner_replace( + node, offset, result.node); + node_t::owned(result.node, e); + return {result.node, result.added, false}; + } + IMMER_CATCH (...) { + node_t::delete_deep_shift(result.node, shift + B); + IMMER_RETHROW; + } + } + } else if (node->datamap() & bit) { + auto offset = node->data_count(bit); + auto val = node->values() + offset; + if (Equal{}(*val, v)) { + if (node->can_mutate(e)) { + auto vals = node->ensure_mutable_values(e); + vals[offset] = std::move(v); + return {node, false, true}; + } else { + auto r = node_t::copy_inner_replace_value( + node, offset, std::move(v)); + return {node_t::owned_values(r, e), false, false}; + } + } else { + auto mutate = node->can_mutate(e); + auto mutate_values = mutate && node->can_mutate_values(e); + auto hash2 = Hash{}(*val); + auto child = node_t::make_merged_e( + e, + shift + B, + std::move(v), + hash, + mutate_values ? std::move(*val) : *val, + hash2); + IMMER_TRY { + auto r = mutate ? node_t::move_inner_replace_merged( + e, node, bit, offset, child) + : node_t::copy_inner_replace_merged( + node, bit, offset, child); + return {node_t::owned_values_safe(r, e), true, mutate}; + } + IMMER_CATCH (...) { + node_t::delete_deep_shift(child, shift + B); + IMMER_RETHROW; + } + } + } else { + auto mutate = node->can_mutate(e); + auto r = mutate ? node_t::move_inner_insert_value( + e, node, bit, std::move(v)) + : node_t::copy_inner_insert_value( + node, bit, std::move(v)); + return {node_t::owned_values(r, e), true, mutate}; + } + } + } + + void add_mut(edit_t e, T v) { auto hash = Hash{}(v); - auto res = do_add(root, std::move(v), hash, 0); - auto new_size = size + (res.second ? 1 : 0); - return { res.first, new_size }; + auto res = do_add_mut(e, root, std::move(v), hash, 0); + if (!res.mutated && root->dec()) + node_t::delete_deep(root, 0); + root = res.node; + size += res.added ? 1 : 0; } - template - std::pair - do_update(node_t* node, K&& k, Fn&& fn, - hash_t hash, shift_t shift) const + using update_result = add_result; + + template + update_result + do_update(node_t* node, K&& k, Fn&& fn, hash_t hash, shift_t shift) const { if (shift == max_shift) { auto fst = node->collisions(); auto lst = fst + node->collision_count(); for (; fst != lst; ++fst) if (Equal{}(*fst, k)) - return { - node_t::copy_collision_replace( - node, fst, Combine{}(std::forward(k), - std::forward(fn)( - Project{}(*fst)))), - false - }; - return { - node_t::copy_collision_insert( - node, Combine{}(std::forward(k), - std::forward(fn)( - Default{}()))), - true - }; + return {node_t::copy_collision_replace( + node, + fst, + Combine{}(std::forward(k), + std::forward(fn)(Project{}( + detail::as_const(*fst))))), + false}; + return {node_t::copy_collision_insert( + node, + Combine{}(std::forward(k), + std::forward(fn)(Default{}()))), + true}; } else { auto idx = (hash & (mask << shift)) >> shift; auto bit = bitmap_t{1u} << idx; if (node->nodemap() & bit) { - auto offset = popcount(node->nodemap() & (bit - 1)); + auto offset = node->children_count(bit); auto result = do_update( - node->children() [offset], k, std::forward(fn), - hash, shift + B); - try { - result.first = node_t::copy_inner_replace( - node, offset, result.first); + node->children()[offset], + k, + std::forward(fn), + hash, + shift + B); + IMMER_TRY { + result.node = + node_t::copy_inner_replace(node, offset, result.node); return result; - } catch (...) { - node_t::delete_deep_shift(result.first, shift + B); - throw; + } + IMMER_CATCH (...) { + node_t::delete_deep_shift(result.node, shift + B); + IMMER_RETHROW; } } else if (node->datamap() & bit) { - auto offset = popcount(node->datamap() & (bit - 1)); + auto offset = node->data_count(bit); auto val = node->values() + offset; if (Equal{}(*val, k)) - return { - node_t::copy_inner_replace_value( - node, offset, Combine{}(std::forward(k), - std::forward(fn)( - Project{}(*val)))), - false - }; + return {node_t::copy_inner_replace_value( + node, + offset, + Combine{}(std::forward(k), + std::forward(fn)(Project{}( + detail::as_const(*val))))), + false}; else { auto child = node_t::make_merged( - shift + B, Combine{}(std::forward(k), - std::forward(fn)( - Default{}())), - hash, *val, Hash{}(*val)); - try { - return { - node_t::copy_inner_replace_merged( - node, bit, offset, child), - true - }; - } catch (...) { + shift + B, + Combine{}(std::forward(k), + std::forward(fn)(Default{}())), + hash, + *val, + Hash{}(*val)); + IMMER_TRY { + return {node_t::copy_inner_replace_merged( + node, bit, offset, child), + true}; + } + IMMER_CATCH (...) { node_t::delete_deep_shift(child, shift + B); - throw; + IMMER_RETHROW; } } } else { - return { - node_t::copy_inner_insert_value( - node, bit, Combine{}(std::forward(k), - std::forward(fn)( - Default{}()))), - true - }; + return {node_t::copy_inner_insert_value( + node, + bit, + Combine{}(std::forward(k), + std::forward(fn)(Default{}()))), + true}; } } } - template + template champ update(const K& k, Fn&& fn) const { auto hash = Hash{}(k); - auto res = do_update( + auto res = do_update( + root, k, std::forward(fn), hash, 0); + auto new_size = size + (res.added ? 1 : 0); + return {res.node, new_size}; + } + + template + node_t* do_update_if_exists( + node_t* node, K&& k, Fn&& fn, hash_t hash, shift_t shift) const + { + if (shift == max_shift) { + auto fst = node->collisions(); + auto lst = fst + node->collision_count(); + for (; fst != lst; ++fst) + if (Equal{}(*fst, k)) + return node_t::copy_collision_replace( + node, + fst, + Combine{}(std::forward(k), + std::forward(fn)( + Project{}(detail::as_const(*fst))))); + return nullptr; + } else { + auto idx = (hash & (mask << shift)) >> shift; + auto bit = bitmap_t{1u} << idx; + if (node->nodemap() & bit) { + auto offset = node->children_count(bit); + auto result = do_update_if_exists( + node->children()[offset], + k, + std::forward(fn), + hash, + shift + B); + IMMER_TRY { + return result ? node_t::copy_inner_replace( + node, offset, result) + : nullptr; + } + IMMER_CATCH (...) { + node_t::delete_deep_shift(result, shift + B); + IMMER_RETHROW; + } + } else if (node->datamap() & bit) { + auto offset = node->data_count(bit); + auto val = node->values() + offset; + if (Equal{}(*val, k)) + return node_t::copy_inner_replace_value( + node, + offset, + Combine{}(std::forward(k), + std::forward(fn)( + Project{}(detail::as_const(*val))))); + else { + return nullptr; + } + } else { + return nullptr; + } + } + } + + template + champ update_if_exists(const K& k, Fn&& fn) const + { + auto hash = Hash{}(k); + auto res = do_update_if_exists( root, k, std::forward(fn), hash, 0); - auto new_size = size + (res.second ? 1 : 0); - return { res.first, new_size }; + if (res) { + return {res, size}; + } else { + return {root->inc(), size}; + }; + } + + using update_mut_result = add_mut_result; + + template + update_mut_result do_update_mut(edit_t e, + node_t* node, + K&& k, + Fn&& fn, + hash_t hash, + shift_t shift) const + { + if (shift == max_shift) { + auto fst = node->collisions(); + auto lst = fst + node->collision_count(); + for (; fst != lst; ++fst) + if (Equal{}(*fst, k)) { + if (node->can_mutate(e)) { + *fst = Combine{}( + std::forward(k), + std::forward(fn)(Project{}(std::move(*fst)))); + return {node, false, true}; + } else { + auto r = node_t::copy_collision_replace( + node, + fst, + Combine{}(std::forward(k), + std::forward(fn)( + Project{}(detail::as_const(*fst))))); + return {node_t::owned(r, e), false, false}; + } + } + auto v = Combine{}(std::forward(k), + std::forward(fn)(Default{}())); + auto mutate = node->can_mutate(e); + auto r = mutate ? node_t::move_collision_insert(node, std::move(v)) + : node_t::copy_collision_insert(node, std::move(v)); + return {node_t::owned(r, e), true, mutate}; + } else { + auto idx = (hash & (mask << shift)) >> shift; + auto bit = bitmap_t{1u} << idx; + if (node->nodemap() & bit) { + auto offset = node->children_count(bit); + auto child = node->children()[offset]; + if (node->can_mutate(e)) { + auto result = do_update_mut( + e, child, k, std::forward(fn), hash, shift + B); + node->children()[offset] = result.node; + if (!result.mutated && child->dec()) + node_t::delete_deep_shift(child, shift + B); + return {node, result.added, true}; + } else { + auto result = do_update( + child, k, std::forward(fn), hash, shift + B); + IMMER_TRY { + result.node = node_t::copy_inner_replace( + node, offset, result.node); + node_t::owned(result.node, e); + return {result.node, result.added, false}; + } + IMMER_CATCH (...) { + node_t::delete_deep_shift(result.node, shift + B); + IMMER_RETHROW; + } + } + } else if (node->datamap() & bit) { + auto offset = node->data_count(bit); + auto val = node->values() + offset; + if (Equal{}(*val, k)) { + if (node->can_mutate(e)) { + auto vals = node->ensure_mutable_values(e); + vals[offset] = Combine{}(std::forward(k), + std::forward(fn)(Project{}( + std::move(vals[offset])))); + return {node, false, true}; + } else { + auto r = node_t::copy_inner_replace_value( + node, + offset, + Combine{}(std::forward(k), + std::forward(fn)( + Project{}(detail::as_const(*val))))); + return {node_t::owned_values(r, e), false, false}; + } + } else { + auto mutate = node->can_mutate(e); + auto mutate_values = mutate && node->can_mutate_values(e); + auto hash2 = Hash{}(*val); + auto child = node_t::make_merged_e( + e, + shift + B, + Combine{}(std::forward(k), + std::forward(fn)(Default{}())), + hash, + mutate_values ? std::move(*val) : *val, + hash2); + IMMER_TRY { + auto r = mutate ? node_t::move_inner_replace_merged( + e, node, bit, offset, child) + : node_t::copy_inner_replace_merged( + node, bit, offset, child); + return {node_t::owned_values_safe(r, e), true, mutate}; + } + IMMER_CATCH (...) { + node_t::delete_deep_shift(child, shift + B); + IMMER_RETHROW; + } + } + } else { + auto mutate = node->can_mutate(e); + auto v = Combine{}(std::forward(k), + std::forward(fn)(Default{}())); + auto r = mutate ? node_t::move_inner_insert_value( + e, node, bit, std::move(v)) + : node_t::copy_inner_insert_value( + node, bit, std::move(v)); + return {node_t::owned_values(r, e), true, mutate}; + } + } + } + + template + void update_mut(edit_t e, const K& k, Fn&& fn) + { + auto hash = Hash{}(k); + auto res = do_update_mut( + e, root, k, std::forward(fn), hash, 0); + if (!res.mutated && root->dec()) + node_t::delete_deep(root, 0); + root = res.node; + size += res.added ? 1 : 0; + } + + struct update_if_exists_mut_result + { + node_t* node; + bool mutated; + }; + + template + update_if_exists_mut_result do_update_if_exists_mut(edit_t e, + node_t* node, + K&& k, + Fn&& fn, + hash_t hash, + shift_t shift) const + { + if (shift == max_shift) { + auto fst = node->collisions(); + auto lst = fst + node->collision_count(); + for (; fst != lst; ++fst) + if (Equal{}(*fst, k)) { + if (node->can_mutate(e)) { + *fst = Combine{}( + std::forward(k), + std::forward(fn)(Project{}(std::move(*fst)))); + return {node, true}; + } else { + auto r = node_t::copy_collision_replace( + node, + fst, + Combine{}(std::forward(k), + std::forward(fn)( + Project{}(detail::as_const(*fst))))); + return {node_t::owned(r, e), false}; + } + } + return {nullptr, false}; + } else { + auto idx = (hash & (mask << shift)) >> shift; + auto bit = bitmap_t{1u} << idx; + if (node->nodemap() & bit) { + auto offset = node->children_count(bit); + auto child = node->children()[offset]; + if (node->can_mutate(e)) { + auto result = do_update_if_exists_mut( + e, child, k, std::forward(fn), hash, shift + B); + if (result.node) { + node->children()[offset] = result.node; + if (!result.mutated && child->dec()) + node_t::delete_deep_shift(child, shift + B); + return {node, true}; + } else { + return {nullptr, false}; + } + } else { + auto result = do_update_if_exists( + child, k, std::forward(fn), hash, shift + B); + IMMER_TRY { + if (result) { + result = node_t::copy_inner_replace( + node, offset, result); + node_t::owned(result, e); + return {result, false}; + } else { + return {nullptr, false}; + } + } + IMMER_CATCH (...) { + node_t::delete_deep_shift(result, shift + B); + IMMER_RETHROW; + } + } + } else if (node->datamap() & bit) { + auto offset = node->data_count(bit); + auto val = node->values() + offset; + if (Equal{}(*val, k)) { + if (node->can_mutate(e)) { + auto vals = node->ensure_mutable_values(e); + vals[offset] = Combine{}(std::forward(k), + std::forward(fn)(Project{}( + std::move(vals[offset])))); + return {node, true}; + } else { + auto r = node_t::copy_inner_replace_value( + node, + offset, + Combine{}(std::forward(k), + std::forward(fn)( + Project{}(detail::as_const(*val))))); + return {node_t::owned_values(r, e), false}; + } + } else { + return {nullptr, false}; + } + } else { + return {nullptr, false}; + } + } + } + + template + void update_if_exists_mut(edit_t e, const K& k, Fn&& fn) + { + auto hash = Hash{}(k); + auto res = do_update_if_exists_mut( + e, root, k, std::forward(fn), hash, 0); + if (res.node) { + if (!res.mutated && root->dec()) + node_t::delete_deep(root, 0); + root = res.node; + } } // basically: @@ -330,20 +1173,30 @@ struct champ union data_t { - T* singleton; + T* singleton; node_t* tree; }; kind_t kind; data_t data; - sub_result() : kind{nothing} {}; - sub_result(T* x) : kind{singleton} { data.singleton = x; }; - sub_result(node_t* x) : kind{tree} { data.tree = x; }; + sub_result() + : kind{nothing} {}; + sub_result(T* x) + : kind{singleton} + { + data.singleton = x; + }; + sub_result(node_t* x) + : kind{tree} + { + data.tree = x; + }; }; template - sub_result do_sub(node_t* node, const K& k, hash_t hash, shift_t shift) const + sub_result + do_sub(node_t* node, const K& k, hash_t hash, shift_t shift) const { if (shift == max_shift) { auto fst = node->collisions(); @@ -351,51 +1204,65 @@ struct champ for (auto cur = fst; cur != lst; ++cur) if (Equal{}(*cur, k)) return node->collision_count() > 2 - ? node_t::copy_collision_remove(node, cur) - : sub_result{fst + (cur == fst)}; + ? node_t::copy_collision_remove(node, cur) + : sub_result{fst + (cur == fst)}; +#if !defined(_MSC_VER) +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#endif +#endif + // Apparently GCC is generating this warning sometimes when + // compiling the benchmarks. It makes however no sense at all. return {}; +#if !defined(_MSC_VER) +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic pop +#endif +#endif } else { auto idx = (hash & (mask << shift)) >> shift; auto bit = bitmap_t{1u} << idx; if (node->nodemap() & bit) { - auto offset = popcount(node->nodemap() & (bit - 1)); - auto result = do_sub(node->children() [offset], - k, hash, shift + B); + auto offset = node->children_count(bit); + auto result = + do_sub(node->children()[offset], k, hash, shift + B); switch (result.kind) { case sub_result::nothing: return {}; case sub_result::singleton: return node->datamap() == 0 && - popcount(node->nodemap()) == 1 && - shift > 0 - ? result - : node_t::copy_inner_replace_inline( - node, bit, offset, *result.data.singleton); + node->children_count() == 1 && shift > 0 + ? result + : node_t::copy_inner_replace_inline( + node, bit, offset, *result.data.singleton); case sub_result::tree: - try { - return node_t::copy_inner_replace(node, offset, - result.data.tree); - } catch (...) { + IMMER_TRY { + return node_t::copy_inner_replace( + node, offset, result.data.tree); + } + IMMER_CATCH (...) { node_t::delete_deep_shift(result.data.tree, shift + B); - throw; + IMMER_RETHROW; } } } else if (node->datamap() & bit) { - auto offset = popcount(node->datamap() & (bit - 1)); + auto offset = node->data_count(bit); auto val = node->values() + offset; if (Equal{}(*val, k)) { - auto nv = popcount(node->datamap()); + auto nv = node->data_count(); if (node->nodemap() || nv > 2) - return node_t::copy_inner_remove_value(node, bit, offset); + return node_t::copy_inner_remove_value( + node, bit, offset); else if (nv == 2) { - return shift > 0 - ? sub_result{node->values() + !offset} - : node_t::make_inner_n(0, - node->datamap() & ~bit, - node->values()[!offset]); + return shift > 0 ? sub_result{node->values() + !offset} + : node_t::make_inner_n( + 0, + node->datamap() & ~bit, + node->values()[!offset]); } else { assert(shift == 0); - return empty().root->inc(); + return empty(); } } } @@ -407,21 +1274,225 @@ struct champ champ sub(const K& k) const { auto hash = Hash{}(k); - auto res = do_sub(root, k, hash, 0); + auto res = do_sub(root, k, hash, 0); switch (res.kind) { case sub_result::nothing: return *this; case sub_result::tree: - return { - res.data.tree, - size - 1 - }; + return {res.data.tree, size - 1}; + default: + IMMER_UNREACHABLE; + } + } + + struct sub_result_mut + { + using kind_t = typename sub_result::kind_t; + using data_t = typename sub_result::data_t; + + kind_t kind; + data_t data; + bool owned; + bool mutated; + + sub_result_mut(sub_result a) + : kind{a.kind} + , data{a.data} + , owned{false} + , mutated{false} + {} + sub_result_mut(sub_result a, bool m) + : kind{a.kind} + , data{a.data} + , owned{false} + , mutated{m} + {} + sub_result_mut() + : kind{kind_t::nothing} + , mutated{false} {}; + sub_result_mut(T* x, bool m) + : kind{kind_t::singleton} + , owned{m} + , mutated{m} + { + data.singleton = x; + }; + sub_result_mut(T* x, bool o, bool m) + : kind{kind_t::singleton} + , owned{o} + , mutated{m} + { + data.singleton = x; + }; + sub_result_mut(node_t* x, bool m) + : kind{kind_t::tree} + , owned{false} + , mutated{m} + { + data.tree = x; + }; + }; + + template + sub_result_mut do_sub_mut(edit_t e, + node_t* node, + const K& k, + hash_t hash, + shift_t shift, + void* store) const + { + auto mutate = node->can_mutate(e); + if (shift == max_shift) { + auto fst = node->collisions(); + auto lst = fst + node->collision_count(); + for (auto cur = fst; cur != lst; ++cur) { + if (Equal{}(*cur, k)) { + if (node->collision_count() <= 2) { + if (mutate) { + auto r = new (store) + T{std::move(node->collisions()[cur == fst])}; + node_t::delete_collision(node); + return sub_result_mut{r, true}; + } else { + return sub_result_mut{fst + (cur == fst), false}; + } + } else { + auto r = mutate + ? node_t::move_collision_remove(node, cur) + : node_t::copy_collision_remove(node, cur); + return {node_t::owned(r, e), mutate}; + } + } + } + return {}; + } else { + auto idx = (hash & (mask << shift)) >> shift; + auto bit = bitmap_t{1u} << idx; + if (node->nodemap() & bit) { + auto offset = node->children_count(bit); + auto children = node->children(); + auto child = children[offset]; + auto result = + mutate ? do_sub_mut(e, child, k, hash, shift + B, store) + : do_sub(child, k, hash, shift + B); + switch (result.kind) { + case sub_result::nothing: + return {}; + case sub_result::singleton: + if (node->datamap() == 0 && node->children_count() == 1 && + shift > 0) { + if (mutate) { + node_t::delete_inner(node); + if (!result.mutated && child->dec()) + node_t::delete_deep_shift(child, shift + B); + } + return {result.data.singleton, result.owned, mutate}; + } else { + auto r = + mutate ? node_t::move_inner_replace_inline( + e, + node, + bit, + offset, + result.owned + ? std::move(*result.data.singleton) + : *result.data.singleton) + : node_t::copy_inner_replace_inline( + node, + bit, + offset, + *result.data.singleton); + if (result.owned) + detail::destroy_at(result.data.singleton); + if (!result.mutated && mutate && child->dec()) + node_t::delete_deep_shift(child, shift + B); + return {node_t::owned_values(r, e), mutate}; + } + case sub_result::tree: + if (mutate) { + children[offset] = result.data.tree; + if (!result.mutated && child->dec()) + node_t::delete_deep_shift(child, shift + B); + return {node, true}; + } else { + IMMER_TRY { + auto r = node_t::copy_inner_replace( + node, offset, result.data.tree); + return {node_t::owned(r, e), false}; + } + IMMER_CATCH (...) { + node_t::delete_deep_shift(result.data.tree, + shift + B); + IMMER_RETHROW; + } + } + } + } else if (node->datamap() & bit) { + auto offset = node->data_count(bit); + auto val = node->values() + offset; + auto mutate_values = mutate && node->can_mutate_values(e); + if (Equal{}(*val, k)) { + auto nv = node->data_count(); + if (node->nodemap() || nv > 2) { + auto r = mutate ? node_t::move_inner_remove_value( + e, node, bit, offset) + : node_t::copy_inner_remove_value( + node, bit, offset); + return {node_t::owned_values_safe(r, e), mutate}; + } else if (nv == 2) { + if (shift > 0) { + if (mutate_values) { + auto r = new (store) + T{std::move(node->values()[!offset])}; + node_t::delete_inner(node); + return {r, true}; + } else { + return {node->values() + !offset, false}; + } + } else { + auto& v = node->values()[!offset]; + auto r = node_t::make_inner_n( + 0, + node->datamap() & ~bit, + mutate_values ? std::move(v) : v); + assert(!node->nodemap()); + if (mutate) + node_t::delete_inner(node); + return {node_t::owned_values(r, e), mutate}; + } + } else { + assert(shift == 0); + if (mutate) + node_t::delete_inner(node); + return {empty(), mutate}; + } + } + } + return {}; + } + } + + template + void sub_mut(edit_t e, const K& k) + { + auto store = aligned_storage_for{}; + auto hash = Hash{}(k); + auto res = do_sub_mut(e, root, k, hash, 0, &store); + switch (res.kind) { + case sub_result::nothing: + break; + case sub_result::tree: + if (!res.mutated && root->dec()) + node_t::delete_deep(root, 0); + root = res.data.tree; + --size; + break; default: IMMER_UNREACHABLE; } } - template + template bool equals(const champ& other) const { return size == other.size && equals_tree(root, other.root, 0); @@ -435,16 +1506,16 @@ struct champ else if (depth == max_depth) { auto nv = a->collision_count(); return nv == b->collision_count() && - equals_collisions(a->collisions(), b->collisions(), nv); + equals_collisions(a->collisions(), b->collisions(), nv); } else { - if (a->nodemap() != b->nodemap() || - a->datamap() != b->datamap()) + if (a->nodemap() != b->nodemap() || a->datamap() != b->datamap()) return false; - auto n = popcount(a->nodemap()); + auto n = a->children_count(); for (auto i = count_t{}; i < n; ++i) - if (!equals_tree(a->children()[i], b->children()[i], depth + 1)) + if (!equals_tree( + a->children()[i], b->children()[i], depth + 1)) return false; - auto nv = popcount(a->datamap()); + auto nv = a->data_count(); return !nv || equals_values(a->values(), b->values(), nv); } } @@ -465,7 +1536,8 @@ struct champ if (Eq{}(*a, *fst)) goto good; return false; - good: continue; + good: + continue; } return true; } diff --git a/src/immer/detail/hamts/champ_iterator.hpp b/src/immer/detail/hamts/champ_iterator.hpp index 07d552daca..5ff3106024 100644 --- a/src/immer/detail/hamts/champ_iterator.hpp +++ b/src/immer/detail/hamts/champ_iterator.hpp @@ -27,16 +27,17 @@ struct champ_iterator using tree_t = champ; using node_t = typename tree_t::node_t; - struct end_t {}; - champ_iterator() = default; + struct end_t + {}; + champ_iterator(const tree_t& v) - : depth_ { 0 } + : depth_{0} { if (v.root->datamap()) { cur_ = v.root->values(); - end_ = v.root->values() + popcount(v.root->datamap()); + end_ = v.root->values() + v.root->data_count(); } else { cur_ = end_ = nullptr; } @@ -45,17 +46,17 @@ struct champ_iterator } champ_iterator(const tree_t& v, end_t) - : cur_ { nullptr } - , end_ { nullptr } - , depth_ { 0 } + : cur_{nullptr} + , end_{nullptr} + , depth_{0} { path_[0] = &v.root; } champ_iterator(const champ_iterator& other) - : cur_ { other.cur_ } - , end_ { other.end_ } - , depth_ { other.depth_ } + : cur_{other.cur_} + , end_{other.end_} + , depth_{other.depth_} { std::copy(other.path_, other.path_ + depth_ + 1, path_); } @@ -66,7 +67,9 @@ struct champ_iterator T* cur_; T* end_; count_t depth_; - node_t* const* path_[max_depth + 1]; + node_t* const* path_[max_depth + 1] = { + 0, + }; void increment() { @@ -78,14 +81,16 @@ struct champ_iterator { if (depth_ < max_depth) { auto parent = *path_[depth_]; + assert(parent); if (parent->nodemap()) { ++depth_; path_[depth_] = parent->children(); - auto child = *path_[depth_]; + auto child = *path_[depth_]; + assert(child); if (depth_ < max_depth) { if (child->datamap()) { cur_ = child->values(); - end_ = cur_ + popcount(child->datamap()); + end_ = cur_ + child->data_count(); } } else { cur_ = child->collisions(); @@ -101,15 +106,16 @@ struct champ_iterator { while (depth_ > 0) { auto parent = *path_[depth_ - 1]; - auto last = parent->children() + popcount(parent->nodemap()); + auto last = parent->children() + parent->children_count(); auto next = path_[depth_] + 1; if (next < last) { path_[depth_] = next; - auto child = *path_[depth_]; + auto child = *path_[depth_]; + assert(child); if (depth_ < max_depth) { if (child->datamap()) { cur_ = child->values(); - end_ = cur_ + popcount(child->datamap()); + end_ = cur_ + child->data_count(); } } else { cur_ = child->collisions(); @@ -117,7 +123,7 @@ struct champ_iterator } return true; } - -- depth_; + --depth_; } return false; } @@ -137,15 +143,9 @@ struct champ_iterator } } - bool equal(const champ_iterator& other) const - { - return cur_ == other.cur_; - } + bool equal(const champ_iterator& other) const { return cur_ == other.cur_; } - const T& dereference() const - { - return *cur_; - } + const T& dereference() const { return *cur_; } }; } // namespace hamts diff --git a/src/immer/detail/hamts/node.hpp b/src/immer/detail/hamts/node.hpp index 61aa381336..7f7dc8b1fe 100644 --- a/src/immer/detail/hamts/node.hpp +++ b/src/immer/detail/hamts/node.hpp @@ -8,22 +8,23 @@ #pragma once +#include #include -#include #include +#include #include - -#ifdef NDEBUG -#define IMMER_HAMTS_TAGGED_NODE 0 -#else -#define IMMER_HAMTS_TAGGED_NODE 1 -#endif +#include namespace immer { namespace detail { namespace hamts { +// For C++14 support. +// Calling the destructor inline breaks MSVC in some obscure +// corner cases. +template constexpr void destroy_at(T* p) { p->~T(); } + template struct node { - using node_t = node; + using node_t = node; using memory = MemoryPolicy; using heap_policy = typename memory::heap; @@ -60,13 +61,12 @@ struct node aligned_storage_for buffer; }; - using values_t = combine_standard_layout_t< - values_data_t, refs_t>; + using values_t = combine_standard_layout_t; struct inner_t { - bitmap_t nodemap; - bitmap_t datamap; + bitmap_t nodemap; + bitmap_t datamap; values_t* values; aligned_storage_for buffer; }; @@ -79,112 +79,156 @@ struct node struct impl_data_t { -#if IMMER_HAMTS_TAGGED_NODE +#if IMMER_TAGGED_NODE kind_t kind; #endif data_t data; }; - using impl_t = combine_standard_layout_t< - impl_data_t, refs_t>; + using impl_t = combine_standard_layout_t; impl_t impl; constexpr static std::size_t sizeof_values_n(count_t count) { - return immer_offsetof(values_t, d.buffer) - + sizeof(values_data_t::buffer) * count; + return std::max(sizeof(values_t), + immer_offsetof(values_t, d.buffer) + + sizeof(values_data_t::buffer) * count); } constexpr static std::size_t sizeof_collision_n(count_t count) { - return immer_offsetof(impl_t, d.data.collision.buffer) - + sizeof(collision_t::buffer) * count; + return immer_offsetof(impl_t, d.data.collision.buffer) + + sizeof(collision_t::buffer) * count; } constexpr static std::size_t sizeof_inner_n(count_t count) { - return immer_offsetof(impl_t, d.data.inner.buffer) - + sizeof(inner_t::buffer) * count; + return immer_offsetof(impl_t, d.data.inner.buffer) + + sizeof(inner_t::buffer) * count; } -#if IMMER_HAMTS_TAGGED_NODE - kind_t kind() const - { - return impl.d.kind; - } +#if IMMER_TAGGED_NODE + kind_t kind() const { return impl.d.kind; } #endif auto values() { - assert(kind() == kind_t::inner); + IMMER_ASSERT_TAGGED(kind() == kind_t::inner); assert(impl.d.data.inner.values); return (T*) &impl.d.data.inner.values->d.buffer; } auto values() const { - assert(kind() == kind_t::inner); + IMMER_ASSERT_TAGGED(kind() == kind_t::inner); assert(impl.d.data.inner.values); return (const T*) &impl.d.data.inner.values->d.buffer; } auto children() { - assert(kind() == kind_t::inner); + IMMER_ASSERT_TAGGED(kind() == kind_t::inner); return (node_t**) &impl.d.data.inner.buffer; } auto children() const { - assert(kind() == kind_t::inner); + IMMER_ASSERT_TAGGED(kind() == kind_t::inner); return (const node_t* const*) &impl.d.data.inner.buffer; } auto datamap() const { - assert(kind() == kind_t::inner); + IMMER_ASSERT_TAGGED(kind() == kind_t::inner); return impl.d.data.inner.datamap; } auto nodemap() const { - assert(kind() == kind_t::inner); + IMMER_ASSERT_TAGGED(kind() == kind_t::inner); return impl.d.data.inner.nodemap; } + auto data_count() const + { + IMMER_ASSERT_TAGGED(kind() == kind_t::inner); + return popcount(datamap()); + } + + auto data_count(bitmap_t bit) const + { + IMMER_ASSERT_TAGGED(kind() == kind_t::inner); + return popcount(static_cast(datamap() & (bit - 1))); + } + + auto children_count() const + { + IMMER_ASSERT_TAGGED(kind() == kind_t::inner); + return popcount(nodemap()); + } + + auto children_count(bitmap_t bit) const + { + IMMER_ASSERT_TAGGED(kind() == kind_t::inner); + return popcount(static_cast(nodemap() & (bit - 1))); + } + auto collision_count() const { - assert(kind() == kind_t::collision); + IMMER_ASSERT_TAGGED(kind() == kind_t::collision); return impl.d.data.collision.count; } T* collisions() { - assert(kind() == kind_t::collision); - return (T*)&impl.d.data.collision.buffer; + IMMER_ASSERT_TAGGED(kind() == kind_t::collision); + return (T*) &impl.d.data.collision.buffer; } const T* collisions() const { - assert(kind() == kind_t::collision); - return (const T*)&impl.d.data.collision.buffer; + IMMER_ASSERT_TAGGED(kind() == kind_t::collision); + return (const T*) &impl.d.data.collision.buffer; } - static refs_t& refs(const values_t* x) { return auto_const_cast(get(*x)); } + static refs_t& refs(const values_t* x) + { + return auto_const_cast(get(*x)); + } static const ownee_t& ownee(const values_t* x) { return get(*x); } static ownee_t& ownee(values_t* x) { return get(*x); } + static bool can_mutate(values_t* x, edit_t e) + { + return refs(x).unique() || ownee(x).can_mutate(e); + } - static refs_t& refs(const node_t* x) { return auto_const_cast(get(x->impl)); } - static const ownee_t& ownee(const node_t* x) { return get(x->impl); } + static refs_t& refs(const node_t* x) + { + return auto_const_cast(get(x->impl)); + } + static const ownee_t& ownee(const node_t* x) + { + return get(x->impl); + } static ownee_t& ownee(node_t* x) { return get(x->impl); } + bool can_mutate(edit_t e) const + { + return refs(this).unique() || ownee(this).can_mutate(e); + } + bool can_mutate_values(edit_t e) const + { + return can_mutate(impl.d.data.inner.values, e); + } + static node_t* make_inner_n(count_t n) { assert(n <= branches); auto m = heap::allocate(sizeof_inner_n(n)); auto p = new (m) node_t; -#if IMMER_HAMTS_TAGGED_NODE + assert(p == (node_t*) m); +#if IMMER_TAGGED_NODE p->impl.d.kind = node_t::kind_t::inner; #endif p->impl.d.data.inner.nodemap = 0; @@ -208,12 +252,13 @@ struct node assert(nv <= branches); auto p = make_inner_n(n); if (nv) { - try { + IMMER_TRY { p->impl.d.data.inner.values = new (heap::allocate(sizeof_values_n(nv))) values_t{}; - } catch (...) { + } + IMMER_CATCH (...) { deallocate_inner(p, n); - throw; + IMMER_RETHROW; } } return p; @@ -222,47 +267,48 @@ struct node static node_t* make_inner_n(count_t n, count_t idx, node_t* child) { assert(n >= 1); - auto p = make_inner_n(n); + auto p = make_inner_n(n); p->impl.d.data.inner.nodemap = bitmap_t{1u} << idx; - p->children()[0] = child; + p->children()[0] = child; return p; } - static node_t* make_inner_n(count_t n, - bitmap_t bitmap, - T x) + static node_t* make_inner_n(count_t n, bitmap_t bitmap, T x) { - auto p = make_inner_n(n, 1); + auto p = make_inner_n(n, 1); p->impl.d.data.inner.datamap = bitmap; - try { + IMMER_TRY { new (p->values()) T{std::move(x)}; - } catch (...) { + } + IMMER_CATCH (...) { deallocate_inner(p, n, 1); - throw; + IMMER_RETHROW; } return p; } - static node_t* make_inner_n(count_t n, - count_t idx1, T x1, - count_t idx2, T x2) + static node_t* + make_inner_n(count_t n, count_t idx1, T x1, count_t idx2, T x2) { assert(idx1 != idx2); auto p = make_inner_n(n, 2); - p->impl.d.data.inner.datamap = (bitmap_t{1u} << idx1) | (bitmap_t{1u} << idx2); - auto assign = [&] (auto&& x1, auto&& x2) { + p->impl.d.data.inner.datamap = + (bitmap_t{1u} << idx1) | (bitmap_t{1u} << idx2); + auto assign = [&](auto&& x1, auto&& x2) { auto vp = p->values(); - try { + IMMER_TRY { new (vp) T{std::move(x1)}; - try { + IMMER_TRY { new (vp + 1) T{std::move(x2)}; - } catch (...) { - vp->~T(); - throw; } - } catch (...) { + IMMER_CATCH (...) { + immer::detail::hamts::destroy_at(vp); + IMMER_RETHROW; + } + } + IMMER_CATCH (...) { deallocate_inner(p, n, 2); - throw; + IMMER_RETHROW; } }; if (idx1 < idx2) @@ -274,10 +320,9 @@ struct node static node_t* make_collision_n(count_t n) { - assert(n <= branches); auto m = heap::allocate(sizeof_collision_n(n)); auto p = new (m) node_t; -#if IMMER_HAMTS_TAGGED_NODE +#if IMMER_TAGGED_NODE p->impl.d.kind = node_t::kind_t::collision; #endif p->impl.d.data.collision.count = n; @@ -288,337 +333,712 @@ struct node { auto m = heap::allocate(sizeof_collision_n(2)); auto p = new (m) node_t; -#if IMMER_HAMTS_TAGGED_NODE +#if IMMER_TAGGED_NODE p->impl.d.kind = node_t::kind_t::collision; #endif p->impl.d.data.collision.count = 2; - auto cols = p->collisions(); - try { + auto cols = p->collisions(); + IMMER_TRY { new (cols) T{std::move(v1)}; - try { + IMMER_TRY { new (cols + 1) T{std::move(v2)}; - } catch (...) { + } + IMMER_CATCH (...) { cols->~T(); - throw; + IMMER_RETHROW; } - } catch (...) { + } + IMMER_CATCH (...) { deallocate_collision(p, 2); - throw; + IMMER_RETHROW; } return p; } + T* ensure_mutable_values(edit_t e) + { + assert(can_mutate(e)); + auto old = impl.d.data.inner.values; + if (node_t::can_mutate(old, e)) + return values(); + else { + auto nv = data_count(); + auto nxt = new (heap::allocate(sizeof_values_n(nv))) values_t{}; + auto dst = (T*) &nxt->d.buffer; + auto src = values(); + ownee(nxt) = e; + IMMER_TRY { + detail::uninitialized_copy(src, src + nv, dst); + } + IMMER_CATCH (...) { + deallocate_values(nxt, nv); + IMMER_RETHROW; + } + impl.d.data.inner.values = nxt; + if (refs(old).dec()) + delete_values(old, nv); + return dst; + } + } + static node_t* copy_collision_insert(node_t* src, T v) { - assert(src->kind() == kind_t::collision); + IMMER_ASSERT_TAGGED(src->kind() == kind_t::collision); auto n = src->collision_count(); auto dst = make_collision_n(n + 1); auto srcp = src->collisions(); auto dstp = dst->collisions(); - try { + IMMER_TRY { new (dstp) T{std::move(v)}; - try { - std::uninitialized_copy(srcp, srcp + n, dstp + 1); - } catch (...) { + IMMER_TRY { + detail::uninitialized_copy(srcp, srcp + n, dstp + 1); + } + IMMER_CATCH (...) { + dstp->~T(); + IMMER_RETHROW; + } + } + IMMER_CATCH (...) { + deallocate_collision(dst, n + 1); + IMMER_RETHROW; + } + return dst; + } + + static node_t* move_collision_insert(node_t* src, T v) + { + IMMER_ASSERT_TAGGED(src->kind() == kind_t::collision); + auto n = src->collision_count(); + auto dst = make_collision_n(n + 1); + auto srcp = src->collisions(); + auto dstp = dst->collisions(); + IMMER_TRY { + new (dstp) T{std::move(v)}; + IMMER_TRY { + detail::uninitialized_move(srcp, srcp + n, dstp + 1); + } + IMMER_CATCH (...) { dstp->~T(); - throw; + IMMER_RETHROW; } - } catch (...) { + } + IMMER_CATCH (...) { deallocate_collision(dst, n + 1); - throw; + IMMER_RETHROW; } + delete_collision(src); return dst; } static node_t* copy_collision_remove(node_t* src, T* v) { - assert(src->kind() == kind_t::collision); + IMMER_ASSERT_TAGGED(src->kind() == kind_t::collision); assert(src->collision_count() > 1); auto n = src->collision_count(); auto dst = make_collision_n(n - 1); auto srcp = src->collisions(); auto dstp = dst->collisions(); - try { - dstp = std::uninitialized_copy(srcp, v, dstp); - try { - std::uninitialized_copy(v + 1, srcp + n, dstp); - } catch (...) { - destroy(dst->collisions(), dstp); - throw; - } - } catch (...) { + IMMER_TRY { + dstp = detail::uninitialized_copy(srcp, v, dstp); + IMMER_TRY { + detail::uninitialized_copy(v + 1, srcp + n, dstp); + } + IMMER_CATCH (...) { + detail::destroy(dst->collisions(), dstp); + IMMER_RETHROW; + } + } + IMMER_CATCH (...) { deallocate_collision(dst, n - 1); - throw; + IMMER_RETHROW; } return dst; } + static node_t* move_collision_remove(node_t* src, T* v) + { + IMMER_ASSERT_TAGGED(src->kind() == kind_t::collision); + assert(src->collision_count() > 1); + auto n = src->collision_count(); + auto dst = make_collision_n(n - 1); + auto srcp = src->collisions(); + auto dstp = dst->collisions(); + IMMER_TRY { + dstp = detail::uninitialized_move(srcp, v, dstp); + IMMER_TRY { + detail::uninitialized_move(v + 1, srcp + n, dstp); + } + IMMER_CATCH (...) { + detail::destroy(dst->collisions(), dstp); + IMMER_RETHROW; + } + } + IMMER_CATCH (...) { + deallocate_collision(dst, n - 1); + IMMER_RETHROW; + } + delete_collision(src); + return dst; + } + static node_t* copy_collision_replace(node_t* src, T* pos, T v) { - assert(src->kind() == kind_t::collision); + IMMER_ASSERT_TAGGED(src->kind() == kind_t::collision); auto n = src->collision_count(); auto dst = make_collision_n(n); auto srcp = src->collisions(); auto dstp = dst->collisions(); assert(pos >= srcp && pos < srcp + n); - try { + IMMER_TRY { new (dstp) T{std::move(v)}; - try { - dstp = std::uninitialized_copy(srcp, pos, dstp + 1); - try { - std::uninitialized_copy(pos + 1, srcp + n, dstp); - } catch (...) { - destroy(dst->collisions(), dstp); - throw; + IMMER_TRY { + dstp = detail::uninitialized_copy(srcp, pos, dstp + 1); + IMMER_TRY { + detail::uninitialized_copy(pos + 1, srcp + n, dstp); + } + IMMER_CATCH (...) { + detail::destroy(dst->collisions(), dstp); + IMMER_RETHROW; } - } catch (...) { + } + IMMER_CATCH (...) { dst->collisions()->~T(); - throw; + IMMER_RETHROW; } - } catch (...) { + } + IMMER_CATCH (...) { deallocate_collision(dst, n); - throw; + IMMER_RETHROW; } return dst; } - static node_t* copy_inner_replace(node_t* src, - count_t offset, node_t* child) + static node_t* + copy_inner_replace(node_t* src, count_t offset, node_t* child) { - assert(src->kind() == kind_t::inner); - auto n = popcount(src->nodemap()); + IMMER_ASSERT_TAGGED(src->kind() == kind_t::inner); + auto n = src->children_count(); auto dst = make_inner_n(n, src->impl.d.data.inner.values); auto srcp = src->children(); auto dstp = dst->children(); dst->impl.d.data.inner.datamap = src->datamap(); dst->impl.d.data.inner.nodemap = src->nodemap(); - std::uninitialized_copy(srcp, srcp + n, dstp); - inc_nodes(srcp, n); - srcp[offset]->dec_unsafe(); + std::copy(srcp, srcp + n, dstp); + inc_nodes(srcp, offset); + inc_nodes(srcp + offset + 1, n - offset - 1); dstp[offset] = child; return dst; } - static node_t* copy_inner_replace_value(node_t* src, - count_t offset, T v) + static node_t* owned(node_t* n, edit_t e) + { + ownee(n) = e; + return n; + } + + static node_t* owned_values(node_t* n, edit_t e) { - assert(src->kind() == kind_t::inner); - assert(offset < popcount(src->datamap())); - auto n = popcount(src->nodemap()); - auto nv = popcount(src->datamap()); - auto dst = make_inner_n(n, nv); + ownee(n) = e; + ownee(n->impl.d.data.inner.values) = e; + return n; + } + + static node_t* owned_values_safe(node_t* n, edit_t e) + { + ownee(n) = e; + if (n->impl.d.data.inner.values) + ownee(n->impl.d.data.inner.values) = e; + return n; + } + + static node_t* copy_inner_replace_value(node_t* src, count_t offset, T v) + { + IMMER_ASSERT_TAGGED(src->kind() == kind_t::inner); + assert(offset < src->data_count()); + auto n = src->children_count(); + auto nv = src->data_count(); + auto dst = make_inner_n(n, nv); dst->impl.d.data.inner.datamap = src->datamap(); dst->impl.d.data.inner.nodemap = src->nodemap(); - try { - std::uninitialized_copy( + IMMER_TRY { + detail::uninitialized_copy( src->values(), src->values() + nv, dst->values()); - try { + IMMER_TRY { dst->values()[offset] = std::move(v); - } catch (...) { - destroy_n(dst->values(), nv); - throw; } - } catch (...) { + IMMER_CATCH (...) { + detail::destroy_n(dst->values(), nv); + IMMER_RETHROW; + } + } + IMMER_CATCH (...) { deallocate_inner(dst, n, nv); - throw; + IMMER_RETHROW; } inc_nodes(src->children(), n); - std::uninitialized_copy( - src->children(), src->children() + n, dst->children()); + std::copy(src->children(), src->children() + n, dst->children()); return dst; } - static node_t* copy_inner_replace_merged( - node_t* src, bitmap_t bit, count_t voffset, node_t* node) + static node_t* copy_inner_replace_merged(node_t* src, + bitmap_t bit, + count_t voffset, + node_t* node) { - assert(src->kind() == kind_t::inner); + IMMER_ASSERT_TAGGED(src->kind() == kind_t::inner); assert(!(src->nodemap() & bit)); assert(src->datamap() & bit); - assert(voffset == popcount(src->datamap() & (bit - 1))); - auto n = popcount(src->nodemap()); - auto nv = popcount(src->datamap()); - auto dst = make_inner_n(n + 1, nv - 1); - auto noffset = popcount(src->nodemap() & (bit - 1)); + assert(voffset == src->data_count(bit)); + auto n = src->children_count(); + auto nv = src->data_count(); + auto dst = make_inner_n(n + 1, nv - 1); + auto noffset = src->children_count(bit); dst->impl.d.data.inner.datamap = src->datamap() & ~bit; dst->impl.d.data.inner.nodemap = src->nodemap() | bit; if (nv > 1) { - try { - std::uninitialized_copy( - src->values(), src->values() + voffset, - dst->values()); - try { - std::uninitialized_copy( - src->values() + voffset + 1, src->values() + nv, - dst->values() + voffset); - } catch (...) { - destroy_n(dst->values(), voffset); - throw; + IMMER_TRY { + detail::uninitialized_copy( + src->values(), src->values() + voffset, dst->values()); + IMMER_TRY { + detail::uninitialized_copy(src->values() + voffset + 1, + src->values() + nv, + dst->values() + voffset); + } + IMMER_CATCH (...) { + detail::destroy_n(dst->values(), voffset); + IMMER_RETHROW; } - } catch (...) { + } + IMMER_CATCH (...) { deallocate_inner(dst, n + 1, nv - 1); - throw; + IMMER_RETHROW; } } inc_nodes(src->children(), n); - std::uninitialized_copy( - src->children(), src->children() + noffset, - dst->children()); - std::uninitialized_copy( - src->children() + noffset, src->children() + n, - dst->children() + noffset + 1); + std::copy(src->children(), src->children() + noffset, dst->children()); + std::copy(src->children() + noffset, + src->children() + n, + dst->children() + noffset + 1); + dst->children()[noffset] = node; + return dst; + } + + static node_t* move_inner_replace_merged( + edit_t e, node_t* src, bitmap_t bit, count_t voffset, node_t* node) + { + IMMER_ASSERT_TAGGED(src->kind() == kind_t::inner); + assert(!(src->nodemap() & bit)); + assert(src->datamap() & bit); + assert(voffset == src->data_count(bit)); + auto n = src->children_count(); + auto nv = src->data_count(); + auto dst = make_inner_n(n + 1, nv - 1); + auto noffset = src->children_count(bit); + dst->impl.d.data.inner.datamap = src->datamap() & ~bit; + dst->impl.d.data.inner.nodemap = src->nodemap() | bit; + if (nv > 1) { + auto mutate_values = can_mutate(src->impl.d.data.inner.values, e); + IMMER_TRY { + if (mutate_values) + detail::uninitialized_move( + src->values(), src->values() + voffset, dst->values()); + else + detail::uninitialized_copy( + src->values(), src->values() + voffset, dst->values()); + IMMER_TRY { + if (mutate_values) + detail::uninitialized_move(src->values() + voffset + 1, + src->values() + nv, + dst->values() + voffset); + else + detail::uninitialized_copy(src->values() + voffset + 1, + src->values() + nv, + dst->values() + voffset); + } + IMMER_CATCH (...) { + detail::destroy_n(dst->values(), voffset); + IMMER_RETHROW; + } + } + IMMER_CATCH (...) { + deallocate_inner(dst, n + 1, nv - 1); + IMMER_RETHROW; + } + } + // inc_nodes(src->children(), n); + std::copy(src->children(), src->children() + noffset, dst->children()); + std::copy(src->children() + noffset, + src->children() + n, + dst->children() + noffset + 1); dst->children()[noffset] = node; + delete_inner(src); return dst; } - static node_t* copy_inner_replace_inline( - node_t* src, bitmap_t bit, count_t noffset, T value) + static node_t* copy_inner_replace_inline(node_t* src, + bitmap_t bit, + count_t noffset, + T value) { - assert(src->kind() == kind_t::inner); + IMMER_ASSERT_TAGGED(src->kind() == kind_t::inner); assert(!(src->datamap() & bit)); assert(src->nodemap() & bit); - assert(noffset == popcount(src->nodemap() & (bit - 1))); - auto n = popcount(src->nodemap()); - auto nv = popcount(src->datamap()); - auto dst = make_inner_n(n - 1, nv + 1); - auto voffset = popcount(src->datamap() & (bit - 1)); + assert(noffset == src->children_count(bit)); + auto n = src->children_count(); + auto nv = src->data_count(); + auto dst = make_inner_n(n - 1, nv + 1); + auto voffset = src->data_count(bit); dst->impl.d.data.inner.nodemap = src->nodemap() & ~bit; dst->impl.d.data.inner.datamap = src->datamap() | bit; - try { + IMMER_TRY { if (nv) - std::uninitialized_copy( - src->values(), src->values() + voffset, - dst->values()); - try { + detail::uninitialized_copy( + src->values(), src->values() + voffset, dst->values()); + IMMER_TRY { new (dst->values() + voffset) T{std::move(value)}; - try { + IMMER_TRY { if (nv) - std::uninitialized_copy( - src->values() + voffset, src->values() + nv, - dst->values() + voffset + 1); - } catch (...) { + detail::uninitialized_copy(src->values() + voffset, + src->values() + nv, + dst->values() + voffset + 1); + } + IMMER_CATCH (...) { dst->values()[voffset].~T(); - throw; + IMMER_RETHROW; } - } catch (...) { - destroy_n(dst->values(), voffset); - throw; } - } catch (...) { + IMMER_CATCH (...) { + detail::destroy_n(dst->values(), voffset); + IMMER_RETHROW; + } + } + IMMER_CATCH (...) { deallocate_inner(dst, n - 1, nv + 1); - throw; + IMMER_RETHROW; } - inc_nodes(src->children(), n); - src->children()[noffset]->dec_unsafe(); - std::uninitialized_copy( - src->children(), src->children() + noffset, - dst->children()); - std::uninitialized_copy( - src->children() + noffset + 1, src->children() + n, - dst->children() + noffset); + inc_nodes(src->children(), noffset); + inc_nodes(src->children() + noffset + 1, n - noffset - 1); + std::copy(src->children(), src->children() + noffset, dst->children()); + std::copy(src->children() + noffset + 1, + src->children() + n, + dst->children() + noffset); return dst; } - static node_t* copy_inner_remove_value( - node_t* src, bitmap_t bit, count_t voffset) + static node_t* move_inner_replace_inline( + edit_t e, node_t* src, bitmap_t bit, count_t noffset, T value) { - assert(src->kind() == kind_t::inner); + IMMER_ASSERT_TAGGED(src->kind() == kind_t::inner); + assert(!(src->datamap() & bit)); + assert(src->nodemap() & bit); + assert(noffset == src->children_count(bit)); + auto n = src->children_count(); + auto nv = src->data_count(); + auto dst = make_inner_n(n - 1, nv + 1); + auto voffset = src->data_count(bit); + dst->impl.d.data.inner.nodemap = src->nodemap() & ~bit; + dst->impl.d.data.inner.datamap = src->datamap() | bit; + IMMER_TRY { + auto mutate_values = + nv && can_mutate(src->impl.d.data.inner.values, e); + if (nv) { + if (mutate_values) + detail::uninitialized_move( + src->values(), src->values() + voffset, dst->values()); + else + detail::uninitialized_copy( + src->values(), src->values() + voffset, dst->values()); + } + IMMER_TRY { + new (dst->values() + voffset) T{std::move(value)}; + IMMER_TRY { + if (nv) { + if (mutate_values) + detail::uninitialized_move(src->values() + voffset, + src->values() + nv, + dst->values() + voffset + + 1); + else + detail::uninitialized_copy(src->values() + voffset, + src->values() + nv, + dst->values() + voffset + + 1); + } + } + IMMER_CATCH (...) { + dst->values()[voffset].~T(); + IMMER_RETHROW; + } + } + IMMER_CATCH (...) { + detail::destroy_n(dst->values(), voffset); + IMMER_RETHROW; + } + } + IMMER_CATCH (...) { + deallocate_inner(dst, n - 1, nv + 1); + IMMER_RETHROW; + } + std::copy(src->children(), src->children() + noffset, dst->children()); + std::copy(src->children() + noffset + 1, + src->children() + n, + dst->children() + noffset); + delete_inner(src); + return dst; + } + + static node_t* + copy_inner_remove_value(node_t* src, bitmap_t bit, count_t voffset) + { + IMMER_ASSERT_TAGGED(src->kind() == kind_t::inner); assert(!(src->nodemap() & bit)); assert(src->datamap() & bit); - assert(voffset == popcount(src->datamap() & (bit - 1))); - auto n = popcount(src->nodemap()); - auto nv = popcount(src->datamap()); - auto dst = make_inner_n(n, nv - 1); + assert(voffset == src->data_count(bit)); + auto n = src->children_count(); + auto nv = src->data_count(); + auto dst = make_inner_n(n, nv - 1); dst->impl.d.data.inner.datamap = src->datamap() & ~bit; dst->impl.d.data.inner.nodemap = src->nodemap(); if (nv > 1) { - try { - std::uninitialized_copy( - src->values(), src->values() + voffset, - dst->values()); - try { - std::uninitialized_copy( - src->values() + voffset + 1, src->values() + nv, - dst->values() + voffset); - } catch (...) { - destroy_n(dst->values(), voffset); - throw; + IMMER_TRY { + detail::uninitialized_copy( + src->values(), src->values() + voffset, dst->values()); + IMMER_TRY { + detail::uninitialized_copy(src->values() + voffset + 1, + src->values() + nv, + dst->values() + voffset); + } + IMMER_CATCH (...) { + detail::destroy_n(dst->values(), voffset); + IMMER_RETHROW; } - } catch (...) { + } + IMMER_CATCH (...) { deallocate_inner(dst, n, nv - 1); - throw; + IMMER_RETHROW; } } inc_nodes(src->children(), n); - std::uninitialized_copy( - src->children(), src->children() + n, dst->children()); + std::copy(src->children(), src->children() + n, dst->children()); + return dst; + } + + static node_t* move_inner_remove_value(edit_t e, + node_t* src, + bitmap_t bit, + count_t voffset) + { + IMMER_ASSERT_TAGGED(src->kind() == kind_t::inner); + assert(!(src->nodemap() & bit)); + assert(src->datamap() & bit); + assert(voffset == src->data_count(bit)); + auto n = src->children_count(); + auto nv = src->data_count(); + auto dst = make_inner_n(n, nv - 1); + dst->impl.d.data.inner.datamap = src->datamap() & ~bit; + dst->impl.d.data.inner.nodemap = src->nodemap(); + if (nv > 1) { + auto mutate_values = can_mutate(src->impl.d.data.inner.values, e); + if (mutate_values) { + IMMER_TRY { + detail::uninitialized_move( + src->values(), src->values() + voffset, dst->values()); + IMMER_TRY { + detail::uninitialized_move(src->values() + voffset + 1, + src->values() + nv, + dst->values() + voffset); + } + IMMER_CATCH (...) { + detail::destroy_n(dst->values(), voffset); + IMMER_RETHROW; + } + } + IMMER_CATCH (...) { + deallocate_inner(dst, n, nv - 1); + IMMER_RETHROW; + } + } else { + IMMER_TRY { + detail::uninitialized_copy( + src->values(), src->values() + voffset, dst->values()); + IMMER_TRY { + detail::uninitialized_copy(src->values() + voffset + 1, + src->values() + nv, + dst->values() + voffset); + } + IMMER_CATCH (...) { + detail::destroy_n(dst->values(), voffset); + IMMER_RETHROW; + } + } + IMMER_CATCH (...) { + deallocate_inner(dst, n, nv - 1); + IMMER_RETHROW; + } + } + } + std::copy(src->children(), src->children() + n, dst->children()); + delete_inner(src); return dst; } static node_t* copy_inner_insert_value(node_t* src, bitmap_t bit, T v) { - assert(src->kind() == kind_t::inner); - auto n = popcount(src->nodemap()); - auto nv = popcount(src->datamap()); - auto offset = popcount(src->datamap() & (bit - 1)); - auto dst = make_inner_n(n, nv + 1); + IMMER_ASSERT_TAGGED(src->kind() == kind_t::inner); + auto n = src->children_count(); + auto nv = src->data_count(); + auto offset = src->data_count(bit); + auto dst = make_inner_n(n, nv + 1); dst->impl.d.data.inner.datamap = src->datamap() | bit; dst->impl.d.data.inner.nodemap = src->nodemap(); - try { + IMMER_TRY { if (nv) - std::uninitialized_copy( + detail::uninitialized_copy( src->values(), src->values() + offset, dst->values()); - try { + IMMER_TRY { new (dst->values() + offset) T{std::move(v)}; - try { + IMMER_TRY { if (nv) - std::uninitialized_copy( - src->values() + offset, src->values() + nv, - dst->values() + offset + 1); - } catch (...) { + detail::uninitialized_copy(src->values() + offset, + src->values() + nv, + dst->values() + offset + 1); + } + IMMER_CATCH (...) { dst->values()[offset].~T(); - throw; + IMMER_RETHROW; } - } catch (...) { - destroy_n(dst->values(), offset); - throw; } - } catch (...) { + IMMER_CATCH (...) { + detail::destroy_n(dst->values(), offset); + IMMER_RETHROW; + } + } + IMMER_CATCH (...) { deallocate_inner(dst, n, nv + 1); - throw; + IMMER_RETHROW; } inc_nodes(src->children(), n); - std::uninitialized_copy( - src->children(), src->children() + n, dst->children()); + std::copy(src->children(), src->children() + n, dst->children()); + return dst; + } + + static node_t* + move_inner_insert_value(edit_t e, node_t* src, bitmap_t bit, T v) + { + IMMER_ASSERT_TAGGED(src->kind() == kind_t::inner); + auto n = src->children_count(); + auto nv = src->data_count(); + auto offset = src->data_count(bit); + auto dst = make_inner_n(n, nv + 1); + dst->impl.d.data.inner.datamap = src->datamap() | bit; + dst->impl.d.data.inner.nodemap = src->nodemap(); + IMMER_TRY { + auto mutate_values = + nv && can_mutate(src->impl.d.data.inner.values, e); + if (nv) { + if (mutate_values) + detail::uninitialized_move( + src->values(), src->values() + offset, dst->values()); + else + detail::uninitialized_copy( + src->values(), src->values() + offset, dst->values()); + } + IMMER_TRY { + new (dst->values() + offset) T{std::move(v)}; + IMMER_TRY { + if (nv) { + if (mutate_values) + detail::uninitialized_move(src->values() + offset, + src->values() + nv, + dst->values() + offset + + 1); + else + detail::uninitialized_copy(src->values() + offset, + src->values() + nv, + dst->values() + offset + + 1); + } + } + IMMER_CATCH (...) { + dst->values()[offset].~T(); + IMMER_RETHROW; + } + } + IMMER_CATCH (...) { + detail::destroy_n(dst->values(), offset); + IMMER_RETHROW; + } + } + IMMER_CATCH (...) { + deallocate_inner(dst, n, nv + 1); + IMMER_RETHROW; + } + std::copy(src->children(), src->children() + n, dst->children()); + delete_inner(src); return dst; } - static node_t* make_merged(shift_t shift, - T v1, hash_t hash1, - T v2, hash_t hash2) + static node_t* + make_merged(shift_t shift, T v1, hash_t hash1, T v2, hash_t hash2) { if (shift < max_shift) { auto idx1 = hash1 & (mask << shift); auto idx2 = hash2 & (mask << shift); if (idx1 == idx2) { - auto merged = make_merged(shift + B, - std::move(v1), hash1, - std::move(v2), hash2); - try { - return make_inner_n(1, idx1 >> shift, merged); - } catch (...) { + auto merged = make_merged( + shift + B, std::move(v1), hash1, std::move(v2), hash2); + IMMER_TRY { + return make_inner_n( + 1, static_cast(idx1 >> shift), merged); + } + IMMER_CATCH (...) { delete_deep_shift(merged, shift + B); - throw; + IMMER_RETHROW; } } else { return make_inner_n(0, - idx1 >> shift, std::move(v1), - idx2 >> shift, std::move(v2)); + static_cast(idx1 >> shift), + std::move(v1), + static_cast(idx2 >> shift), + std::move(v2)); } } else { return make_collision(std::move(v1), std::move(v2)); } } + static node_t* make_merged_e( + edit_t e, shift_t shift, T v1, hash_t hash1, T v2, hash_t hash2) + { + if (shift < max_shift) { + auto idx1 = hash1 & (mask << shift); + auto idx2 = hash2 & (mask << shift); + if (idx1 == idx2) { + auto merged = make_merged_e( + e, shift + B, std::move(v1), hash1, std::move(v2), hash2); + IMMER_TRY { + return owned( + make_inner_n( + 1, static_cast(idx1 >> shift), merged), + e); + } + IMMER_CATCH (...) { + delete_deep_shift(merged, shift + B); + IMMER_RETHROW; + } + } else { + auto r = make_inner_n(0, + static_cast(idx1 >> shift), + std::move(v1), + static_cast(idx2 >> shift), + std::move(v2)); + return owned_values(r, e); + } + } else { + return owned(make_collision(std::move(v1), std::move(v2)), e); + } + } + node_t* inc() { refs(this).inc(); @@ -632,7 +1052,6 @@ struct node } bool dec() const { return refs(this).dec(); } - void dec_unsafe() const { refs(this).dec_unsafe(); } static void inc_nodes(node_t** p, count_t n) { @@ -643,24 +1062,26 @@ struct node static void delete_values(values_t* p, count_t n) { assert(p); + detail::destroy_n((T*) &p->d.buffer, n); deallocate_values(p, n); } static void delete_inner(node_t* p) { assert(p); - assert(p->kind() == kind_t::inner); + IMMER_ASSERT_TAGGED(p->kind() == kind_t::inner); auto vp = p->impl.d.data.inner.values; if (vp && refs(vp).dec()) - delete_values(vp, popcount(p->datamap())); - deallocate_inner(p, popcount(p->nodemap())); + delete_values(vp, p->data_count()); + deallocate_inner(p, p->children_count()); } static void delete_collision(node_t* p) { assert(p); - assert(p->kind() == kind_t::collision); + IMMER_ASSERT_TAGGED(p->kind() == kind_t::collision); auto n = p->collision_count(); + detail::destroy_n(p->collisions(), n); deallocate_collision(p, n); } @@ -670,7 +1091,7 @@ struct node delete_collision(p); else { auto fst = p->children(); - auto lst = fst + popcount(p->nodemap()); + auto lst = fst + p->children_count(); for (; fst != lst; ++fst) if ((*fst)->dec()) delete_deep(*fst, s + 1); @@ -684,7 +1105,7 @@ struct node delete_collision(p); else { auto fst = p->children(); - auto lst = fst + popcount(p->nodemap()); + auto lst = fst + p->children_count(); for (; fst != lst; ++fst) if ((*fst)->dec()) delete_deep_shift(*fst, s + B); @@ -694,13 +1115,11 @@ struct node static void deallocate_values(values_t* p, count_t n) { - destroy_n((T*) &p->d.buffer, n); heap::deallocate(node_t::sizeof_values_n(n), p); } static void deallocate_collision(node_t* p, count_t n) { - destroy_n(p->collisions(), n); heap::deallocate(node_t::sizeof_collision_n(n), p); } @@ -712,7 +1131,8 @@ struct node static void deallocate_inner(node_t* p, count_t n, count_t nv) { assert(nv); - heap::deallocate(node_t::sizeof_values_n(nv), p->impl.d.data.inner.values); + heap::deallocate(node_t::sizeof_values_n(nv), + p->impl.d.data.inner.values); heap::deallocate(node_t::sizeof_inner_n(n), p); } }; diff --git a/src/immer/detail/iterator_facade.hpp b/src/immer/detail/iterator_facade.hpp index 985b2f17ae..1d5578c875 100644 --- a/src/immer/detail/iterator_facade.hpp +++ b/src/immer/detail/iterator_facade.hpp @@ -19,27 +19,39 @@ struct iterator_core_access { template static decltype(auto) dereference(T&& x) - { return x.dereference(); } + { + return x.dereference(); + } template static decltype(auto) increment(T&& x) - { return x.increment(); } + { + return x.increment(); + } template static decltype(auto) decrement(T&& x) - { return x.decrement(); } + { + return x.decrement(); + } template static decltype(auto) equal(T1&& x1, T2&& x2) - { return x1.equal(x2); } + { + return x1.equal(x2); + } template static decltype(auto) advance(T&& x, D d) - { return x.advance(d); } + { + return x.advance(d); + } template static decltype(auto) distance_to(T1&& x1, T2&& x2) - { return x1.distance_to(x2); } + { + return x1.distance_to(x2); + } }; /*! @@ -48,16 +60,18 @@ struct iterator_core_access template + typename PointerT = T*> class iterator_facade - : public std::iterator { +public: + using iterator_category = IteratorCategoryT; + using value_type = T; + using difference_type = DifferenceTypeT; + using pointer = PointerT; + using reference = ReferenceT; + protected: using access_t = iterator_core_access; @@ -68,17 +82,6 @@ class iterator_facade std::is_base_of::value; - class reference_proxy - { - friend iterator_facade; - DerivedT iter_; - - reference_proxy(DerivedT iter) - : iter_{std::move(iter)} {} - public: - operator ReferenceT() const { return *iter_; } - }; - const DerivedT& derived() const { static_assert(std::is_base_of::value, @@ -93,27 +96,21 @@ class iterator_facade } public: - ReferenceT operator*() const - { - return access_t::dereference(derived()); - } - PointerT operator->() const - { - return &access_t::dereference(derived()); - } - reference_proxy operator[](DifferenceTypeT n) const + ReferenceT operator*() const { return access_t::dereference(derived()); } + PointerT operator->() const { return &access_t::dereference(derived()); } + ReferenceT operator[](DifferenceTypeT n) const { static_assert(is_random_access, ""); - return derived() + n; + return *(derived() + n); } - bool operator==(const DerivedT& rhs) const + friend bool operator==(const DerivedT& a, const DerivedT& b) { - return access_t::equal(derived(), rhs); + return access_t::equal(a, b); } - bool operator!=(const DerivedT& rhs) const + friend bool operator!=(const DerivedT& a, const DerivedT& b) { - return !access_t::equal(derived(), rhs); + return !access_t::equal(a, b); } DerivedT& operator++() @@ -170,31 +167,31 @@ class iterator_facade auto tmp = derived(); return tmp -= n; } - DifferenceTypeT operator-(const DerivedT& rhs) const + friend DifferenceTypeT operator-(const DerivedT& a, const DerivedT& b) { static_assert(is_random_access, ""); - return access_t::distance_to(rhs, derived()); + return access_t::distance_to(b, a); } - bool operator<(const DerivedT& rhs) const + friend bool operator<(const DerivedT& a, const DerivedT& b) { static_assert(is_random_access, ""); - return access_t::distance_to(derived(), rhs) > 0; + return access_t::distance_to(a, b) > 0; } - bool operator<=(const DerivedT& rhs) const + friend bool operator<=(const DerivedT& a, const DerivedT& b) { static_assert(is_random_access, ""); - return access_t::distance_to(derived(), rhs) >= 0; + return access_t::distance_to(a, b) >= 0; } - bool operator>(const DerivedT& rhs) const + friend bool operator>(const DerivedT& a, const DerivedT& b) { static_assert(is_random_access, ""); - return access_t::distance_to(derived(), rhs) < 0; + return access_t::distance_to(a, b) < 0; } - bool operator>=(const DerivedT& rhs) const + friend bool operator>=(const DerivedT& a, const DerivedT& b) { static_assert(is_random_access, ""); - return access_t::distance_to(derived(), rhs) <= 0; + return access_t::distance_to(a, b) <= 0; } }; diff --git a/src/immer/detail/rbts/bits.hpp b/src/immer/detail/rbts/bits.hpp index 549319ae79..f86b44fb01 100644 --- a/src/immer/detail/rbts/bits.hpp +++ b/src/immer/detail/rbts/bits.hpp @@ -8,6 +8,7 @@ #pragma once +#include #include namespace immer { @@ -19,10 +20,10 @@ using shift_t = std::uint32_t; using count_t = std::uint32_t; using size_t = std::size_t; -template +template constexpr T branches = T{1} << B; -template +template constexpr T mask = branches - 1; template diff --git a/src/immer/detail/rbts/node.hpp b/src/immer/detail/rbts/node.hpp index 229b1d9d48..55a512b03d 100644 --- a/src/immer/detail/rbts/node.hpp +++ b/src/immer/detail/rbts/node.hpp @@ -8,30 +8,22 @@ #pragma once -#include +#include #include -#include #include +#include +#include #include #include #include #include -#ifdef NDEBUG -#define IMMER_RBTS_TAGGED_NODE 0 -#else -#define IMMER_RBTS_TAGGED_NODE 1 -#endif - namespace immer { namespace detail { namespace rbts { -template +template struct node { static constexpr auto bits = B; @@ -57,16 +49,13 @@ struct node struct relaxed_data_t { count_t count; - size_t sizes[branches]; + size_t sizes[branches]; }; using relaxed_data_with_meta_t = - combine_standard_layout_t; + combine_standard_layout_t; - using relaxed_data_no_meta_t = - combine_standard_layout_t; + using relaxed_data_no_meta_t = combine_standard_layout_t; using relaxed_t = std::conditional_t buffer; }; union data_t { inner_t inner; - leaf_t leaf; + leaf_t leaf; }; struct impl_data_t { -#if IMMER_RBTS_TAGGED_NODE +#if IMMER_TAGGED_NODE kind_t kind; #endif data_t data; }; - using impl_t = combine_standard_layout_t< - impl_data_t, refs_t, ownee_t>; + using impl_t = combine_standard_layout_t; impl_t impl; @@ -109,30 +97,29 @@ struct node constexpr static std::size_t sizeof_packed_leaf_n(count_t count) { - return immer_offsetof(impl_t, d.data.leaf.buffer) - + sizeof(leaf_t::buffer) * count; + return immer_offsetof(impl_t, d.data.leaf.buffer) + + sizeof(leaf_t::buffer) * count; } constexpr static std::size_t sizeof_packed_inner_n(count_t count) { - return immer_offsetof(impl_t, d.data.inner.buffer) - + sizeof(inner_t::buffer) * count; + return immer_offsetof(impl_t, d.data.inner.buffer) + + sizeof(inner_t::buffer) * count; } constexpr static std::size_t sizeof_packed_relaxed_n(count_t count) { - return immer_offsetof(relaxed_t, d.sizes) - + sizeof(size_t) * count; + return immer_offsetof(relaxed_t, d.sizes) + sizeof(size_t) * count; } constexpr static std::size_t sizeof_packed_inner_r_n(count_t count) { - return embed_relaxed - ? sizeof_packed_inner_n(count) + sizeof_packed_relaxed_n(count) - : sizeof_packed_inner_n(count); + return embed_relaxed ? sizeof_packed_inner_n(count) + + sizeof_packed_relaxed_n(count) + : sizeof_packed_inner_n(count); } - constexpr static std::size_t max_sizeof_leaf = + constexpr static std::size_t max_sizeof_leaf = sizeof_packed_leaf_n(branches); constexpr static std::size_t max_sizeof_inner = @@ -145,78 +132,98 @@ struct node sizeof_packed_inner_r_n(branches); constexpr static std::size_t sizeof_inner_n(count_t n) - { return keep_headroom ? max_sizeof_inner : sizeof_packed_inner_n(n); } + { + return keep_headroom ? max_sizeof_inner : sizeof_packed_inner_n(n); + } constexpr static std::size_t sizeof_inner_r_n(count_t n) - { return keep_headroom ? max_sizeof_inner_r : sizeof_packed_inner_r_n(n); } + { + return keep_headroom ? max_sizeof_inner_r : sizeof_packed_inner_r_n(n); + } constexpr static std::size_t sizeof_relaxed_n(count_t n) - { return keep_headroom ? max_sizeof_relaxed : sizeof_packed_relaxed_n(n); } + { + return keep_headroom ? max_sizeof_relaxed : sizeof_packed_relaxed_n(n); + } constexpr static std::size_t sizeof_leaf_n(count_t n) - { return keep_headroom ? max_sizeof_leaf : sizeof_packed_leaf_n(n); } - - using heap = typename heap_policy::template - optimized::type; - -#if IMMER_RBTS_TAGGED_NODE - kind_t kind() const { - return impl.d.kind; + return keep_headroom ? max_sizeof_leaf : sizeof_packed_leaf_n(n); } + + using heap = + typename heap_policy::template optimized::type; + +#if IMMER_TAGGED_NODE + kind_t kind() const { return impl.d.kind; } #endif relaxed_t* relaxed() { - assert(kind() == kind_t::inner); + IMMER_ASSERT_TAGGED(kind() == kind_t::inner); return impl.d.data.inner.relaxed; } const relaxed_t* relaxed() const { - assert(kind() == kind_t::inner); + IMMER_ASSERT_TAGGED(kind() == kind_t::inner); return impl.d.data.inner.relaxed; } node_t** inner() { - assert(kind() == kind_t::inner); + IMMER_ASSERT_TAGGED(kind() == kind_t::inner); return reinterpret_cast(&impl.d.data.inner.buffer); } T* leaf() { - assert(kind() == kind_t::leaf); + IMMER_ASSERT_TAGGED(kind() == kind_t::leaf); return reinterpret_cast(&impl.d.data.leaf.buffer); } - static refs_t& refs(const relaxed_t* x) { return auto_const_cast(get(*x)); } + static refs_t& refs(const relaxed_t* x) + { + return auto_const_cast(get(*x)); + } static const ownee_t& ownee(const relaxed_t* x) { return get(*x); } static ownee_t& ownee(relaxed_t* x) { return get(*x); } - static refs_t& refs(const node_t* x) { return auto_const_cast(get(x->impl)); } - static const ownee_t& ownee(const node_t* x) { return get(x->impl); } + static refs_t& refs(const node_t* x) + { + return auto_const_cast(get(x->impl)); + } + static const ownee_t& ownee(const node_t* x) + { + return get(x->impl); + } static ownee_t& ownee(node_t* x) { return get(x->impl); } - static node_t* make_inner_n(count_t n) + static node_t* make_inner_n_into(void* buffer, std::size_t size, count_t n) { assert(n <= branches); - auto m = heap::allocate(sizeof_inner_n(n)); - auto p = new (m) node_t; + assert(size >= sizeof_inner_n(n)); + auto p = new (buffer) node_t; p->impl.d.data.inner.relaxed = nullptr; -#if IMMER_RBTS_TAGGED_NODE +#if IMMER_TAGGED_NODE p->impl.d.kind = node_t::kind_t::inner; #endif return p; } + static node_t* make_inner_n(count_t n) + { + assert(n <= branches); + auto m = heap::allocate(sizeof_inner_n(n)); + return make_inner_n_into(m, sizeof_inner_n(n), n); + } static node_t* make_inner_e(edit_t e) { - auto m = heap::allocate(max_sizeof_inner); - auto p = new (m) node_t; - ownee(p) = e; + auto m = heap::allocate(max_sizeof_inner); + auto p = new (m) node_t; + ownee(p) = e; p->impl.d.data.inner.relaxed = nullptr; -#if IMMER_RBTS_TAGGED_NODE +#if IMMER_TAGGED_NODE p->impl.d.kind = node_t::kind_t::inner; #endif return p; @@ -230,18 +237,19 @@ struct node if (embed_relaxed) { mr = reinterpret_cast(mp) + sizeof_inner_n(n); } else { - try { + IMMER_TRY { mr = heap::allocate(sizeof_relaxed_n(n), norefs_tag{}); - } catch (...) { + } + IMMER_CATCH (...) { heap::deallocate(sizeof_inner_r_n(n), mp); - throw; + IMMER_RETHROW; } } - auto p = new (mp) node_t; - auto r = new (mr) relaxed_t; - r->d.count = 0; + auto p = new (mp) node_t; + auto r = new (mr) relaxed_t; + r->d.count = 0; p->impl.d.data.inner.relaxed = r; -#if IMMER_RBTS_TAGGED_NODE +#if IMMER_TAGGED_NODE p->impl.d.kind = node_t::kind_t::inner; #endif return p; @@ -250,15 +258,14 @@ struct node static node_t* make_inner_sr_n(count_t n, relaxed_t* r) { return static_if( - [&] (auto) { - return node_t::make_inner_r_n(n); - }, - [&] (auto) { - auto p = new (heap::allocate(node_t::sizeof_inner_r_n(n))) node_t; + [&](auto) { return node_t::make_inner_r_n(n); }, + [&](auto) { + auto p = + new (heap::allocate(node_t::sizeof_inner_r_n(n))) node_t; assert(r->d.count >= n); node_t::refs(r).inc(); p->impl.d.data.inner.relaxed = r; -#if IMMER_RBTS_TAGGED_NODE +#if IMMER_TAGGED_NODE p->impl.d.kind = node_t::kind_t::inner; #endif return p; @@ -272,20 +279,21 @@ struct node if (embed_relaxed) { mr = reinterpret_cast(mp) + max_sizeof_inner; } else { - try { + IMMER_TRY { mr = heap::allocate(max_sizeof_relaxed, norefs_tag{}); - } catch (...) { + } + IMMER_CATCH (...) { heap::deallocate(max_sizeof_inner_r, mp); - throw; + IMMER_RETHROW; } } - auto p = new (mp) node_t; - auto r = new (mr) relaxed_t; + auto p = new (mp) node_t; + auto r = new (mr) relaxed_t; ownee(p) = e; - static_if([&](auto){ node_t::ownee(r) = e; }); - r->d.count = 0; + static_if([&](auto) { node_t::ownee(r) = e; }); + r->d.count = 0; p->impl.d.data.inner.relaxed = r; -#if IMMER_RBTS_TAGGED_NODE +#if IMMER_TAGGED_NODE p->impl.d.kind = node_t::kind_t::inner; #endif return p; @@ -294,36 +302,43 @@ struct node static node_t* make_inner_sr_e(edit_t e, relaxed_t* r) { return static_if( - [&] (auto) { - return node_t::make_inner_r_e(e); - }, - [&] (auto) { - auto p = new (heap::allocate(node_t::max_sizeof_inner_r)) node_t; + [&](auto) { return node_t::make_inner_r_e(e); }, + [&](auto) { + auto p = + new (heap::allocate(node_t::max_sizeof_inner_r)) node_t; node_t::refs(r).inc(); p->impl.d.data.inner.relaxed = r; - node_t::ownee(p) = e; -#if IMMER_RBTS_TAGGED_NODE + node_t::ownee(p) = e; +#if IMMER_TAGGED_NODE p->impl.d.kind = node_t::kind_t::inner; #endif return p; }); } - static node_t* make_leaf_n(count_t n) + static node_t* make_leaf_n_into(void* buffer, std::size_t size, count_t n) { assert(n <= branches); - auto p = new (heap::allocate(sizeof_leaf_n(n))) node_t; -#if IMMER_RBTS_TAGGED_NODE + assert(size >= sizeof_leaf_n(n)); + auto p = new (buffer) node_t; +#if IMMER_TAGGED_NODE p->impl.d.kind = node_t::kind_t::leaf; #endif return p; } + static node_t* make_leaf_n(count_t n) + { + assert(n <= branches); + auto m = heap::allocate(sizeof_leaf_n(n)); + return make_leaf_n_into(m, sizeof_leaf_n(n), n); + } + static node_t* make_leaf_e(edit_t e) { - auto p = new (heap::allocate(max_sizeof_leaf)) node_t; + auto p = new (heap::allocate(max_sizeof_leaf)) node_t; ownee(p) = e; -#if IMMER_RBTS_TAGGED_NODE +#if IMMER_TAGGED_NODE p->impl.d.kind = node_t::kind_t::leaf; #endif return p; @@ -332,104 +347,111 @@ struct node static node_t* make_inner_n(count_t n, node_t* x) { assert(n >= 1); - auto p = make_inner_n(n); - p->inner() [0] = x; + auto p = make_inner_n(n); + p->inner()[0] = x; return p; } static node_t* make_inner_n(edit_t n, node_t* x) { assert(n >= 1); - auto p = make_inner_n(n); - p->inner() [0] = x; + auto p = make_inner_n(n); + p->inner()[0] = x; return p; } static node_t* make_inner_n(count_t n, node_t* x, node_t* y) { assert(n >= 2); - auto p = make_inner_n(n); - p->inner() [0] = x; - p->inner() [1] = y; + auto p = make_inner_n(n); + p->inner()[0] = x; + p->inner()[1] = y; return p; } static node_t* make_inner_r_n(count_t n, node_t* x) { assert(n >= 1); - auto p = make_inner_r_n(n); - auto r = p->relaxed(); - p->inner() [0] = x; - r->d.count = 1; + auto p = make_inner_r_n(n); + auto r = p->relaxed(); + p->inner()[0] = x; + r->d.count = 1; return p; } static node_t* make_inner_r_n(count_t n, node_t* x, size_t xs) { assert(n >= 1); - auto p = make_inner_r_n(n); - auto r = p->relaxed(); - p->inner() [0] = x; - r->d.sizes [0] = xs; - r->d.count = 1; + assert(xs); + auto p = make_inner_r_n(n); + auto r = p->relaxed(); + p->inner()[0] = x; + r->d.sizes[0] = xs; + r->d.count = 1; return p; } static node_t* make_inner_r_n(count_t n, node_t* x, node_t* y) { assert(n >= 2); - auto p = make_inner_r_n(n); - auto r = p->relaxed(); - p->inner() [0] = x; - p->inner() [1] = y; - r->d.count = 2; + auto p = make_inner_r_n(n); + auto r = p->relaxed(); + p->inner()[0] = x; + p->inner()[1] = y; + r->d.count = 2; return p; } - static node_t* make_inner_r_n(count_t n, - node_t* x, size_t xs, - node_t* y) + static node_t* make_inner_r_n(count_t n, node_t* x, size_t xs, node_t* y) { assert(n >= 2); - auto p = make_inner_r_n(n); - auto r = p->relaxed(); - p->inner() [0] = x; - p->inner() [1] = y; - r->d.sizes [0] = xs; - r->d.count = 2; + assert(xs); + auto p = make_inner_r_n(n); + auto r = p->relaxed(); + p->inner()[0] = x; + p->inner()[1] = y; + r->d.sizes[0] = xs; + r->d.count = 2; return p; } - static node_t* make_inner_r_n(count_t n, - node_t* x, size_t xs, - node_t* y, size_t ys) + static node_t* + make_inner_r_n(count_t n, node_t* x, size_t xs, node_t* y, size_t ys) { assert(n >= 2); - auto p = make_inner_r_n(n); - auto r = p->relaxed(); - p->inner() [0] = x; - p->inner() [1] = y; - r->d.sizes [0] = xs; - r->d.sizes [1] = xs + ys; - r->d.count = 2; + assert(xs); + assert(ys); + auto p = make_inner_r_n(n); + auto r = p->relaxed(); + p->inner()[0] = x; + p->inner()[1] = y; + r->d.sizes[0] = xs; + r->d.sizes[1] = xs + ys; + r->d.count = 2; return p; } static node_t* make_inner_r_n(count_t n, - node_t* x, size_t xs, - node_t* y, size_t ys, - node_t* z, size_t zs) + node_t* x, + size_t xs, + node_t* y, + size_t ys, + node_t* z, + size_t zs) { assert(n >= 3); - auto p = make_inner_r_n(n); - auto r = p->relaxed(); - p->inner() [0] = x; - p->inner() [1] = y; - p->inner() [2] = z; - r->d.sizes [0] = xs; - r->d.sizes [1] = xs + ys; - r->d.sizes [2] = xs + ys + zs; - r->d.count = 3; + assert(xs); + assert(ys); + assert(zs); + auto p = make_inner_r_n(n); + auto r = p->relaxed(); + p->inner()[0] = x; + p->inner()[1] = y; + p->inner()[2] = z; + r->d.sizes[0] = xs; + r->d.sizes[1] = xs + ys; + r->d.sizes[2] = xs + ys + zs; + r->d.count = 3; return p; } @@ -438,11 +460,12 @@ struct node { assert(n >= 1); auto p = make_leaf_n(n); - try { - new (p->leaf()) T{ std::forward(x) }; - } catch (...) { + IMMER_TRY { + new (p->leaf()) T{std::forward(x)}; + } + IMMER_CATCH (...) { heap::deallocate(node_t::sizeof_leaf_n(n), p); - throw; + IMMER_RETHROW; } return p; } @@ -451,27 +474,29 @@ struct node static node_t* make_leaf_e(edit_t e, U&& x) { auto p = make_leaf_e(e); - try { - new (p->leaf()) T{ std::forward(x) }; - } catch (...) { + IMMER_TRY { + new (p->leaf()) T{std::forward(x)}; + } + IMMER_CATCH (...) { heap::deallocate(node_t::max_sizeof_leaf, p); - throw; + IMMER_RETHROW; } return p; } static node_t* make_path(shift_t shift, node_t* node) { - assert(node->kind() == kind_t::leaf); + IMMER_ASSERT_TAGGED(node->kind() == kind_t::leaf); if (shift == endshift) return node; else { auto n = node_t::make_inner_n(1); - try { - n->inner() [0] = make_path(shift - B, node); - } catch (...) { + IMMER_TRY { + n->inner()[0] = make_path(shift - B, node); + } + IMMER_CATCH (...) { heap::deallocate(node_t::sizeof_inner_n(1), n); - throw; + IMMER_RETHROW; } return n; } @@ -479,16 +504,17 @@ struct node static node_t* make_path_e(edit_t e, shift_t shift, node_t* node) { - assert(node->kind() == kind_t::leaf); + IMMER_ASSERT_TAGGED(node->kind() == kind_t::leaf); if (shift == endshift) return node; else { auto n = node_t::make_inner_e(e); - try { - n->inner() [0] = make_path_e(e, shift - B, node); - } catch (...) { + IMMER_TRY { + n->inner()[0] = make_path_e(e, shift - B, node); + } + IMMER_CATCH (...) { heap::deallocate(node_t::max_sizeof_inner, n); - throw; + IMMER_RETHROW; } return n; } @@ -496,41 +522,54 @@ struct node static node_t* copy_inner(node_t* src, count_t n) { - assert(src->kind() == kind_t::inner); + IMMER_ASSERT_TAGGED(src->kind() == kind_t::inner); auto dst = make_inner_n(n); inc_nodes(src->inner(), n); - std::uninitialized_copy(src->inner(), src->inner() + n, dst->inner()); + std::copy(src->inner(), src->inner() + n, dst->inner()); return dst; } static node_t* copy_inner_n(count_t allocn, node_t* src, count_t n) { assert(allocn >= n); - assert(src->kind() == kind_t::inner); + IMMER_ASSERT_TAGGED(src->kind() == kind_t::inner); auto dst = make_inner_n(allocn); return do_copy_inner(dst, src, n); } static node_t* copy_inner_e(edit_t e, node_t* src, count_t n) { - assert(src->kind() == kind_t::inner); + IMMER_ASSERT_TAGGED(src->kind() == kind_t::inner); auto dst = make_inner_e(e); return do_copy_inner(dst, src, n); } static node_t* do_copy_inner(node_t* dst, node_t* src, count_t n) { - assert(dst->kind() == kind_t::inner); - assert(src->kind() == kind_t::inner); + IMMER_ASSERT_TAGGED(dst->kind() == kind_t::inner); + IMMER_ASSERT_TAGGED(src->kind() == kind_t::inner); auto p = src->inner(); inc_nodes(p, n); - std::uninitialized_copy(p, p + n, dst->inner()); + std::copy(p, p + n, dst->inner()); + return dst; + } + + static node_t* do_copy_inner_replace( + node_t* dst, node_t* src, count_t n, count_t offset, node_t* child) + { + IMMER_ASSERT_TAGGED(dst->kind() == kind_t::inner); + IMMER_ASSERT_TAGGED(src->kind() == kind_t::inner); + auto p = src->inner(); + inc_nodes(p, offset); + inc_nodes(p + offset + 1, n - offset - 1); + std::copy(p, p + n, dst->inner()); + dst->inner()[offset] = child; return dst; } static node_t* copy_inner_r(node_t* src, count_t n) { - assert(src->kind() == kind_t::inner); + IMMER_ASSERT_TAGGED(src->kind() == kind_t::inner); auto dst = make_inner_r_n(n); return do_copy_inner_r(dst, src, n); } @@ -538,29 +577,29 @@ struct node static node_t* copy_inner_r_n(count_t allocn, node_t* src, count_t n) { assert(allocn >= n); - assert(src->kind() == kind_t::inner); + IMMER_ASSERT_TAGGED(src->kind() == kind_t::inner); auto dst = make_inner_r_n(allocn); return do_copy_inner_r(dst, src, n); } static node_t* copy_inner_r_e(edit_t e, node_t* src, count_t n) { - assert(src->kind() == kind_t::inner); + IMMER_ASSERT_TAGGED(src->kind() == kind_t::inner); auto dst = make_inner_r_e(e); return do_copy_inner_r(dst, src, n); } static node_t* copy_inner_sr_e(edit_t e, node_t* src, count_t n) { - assert(src->kind() == kind_t::inner); + IMMER_ASSERT_TAGGED(src->kind() == kind_t::inner); auto dst = make_inner_sr_e(e, src->relaxed()); return do_copy_inner_sr(dst, src, n); } static node_t* do_copy_inner_r(node_t* dst, node_t* src, count_t n) { - assert(dst->kind() == kind_t::inner); - assert(src->kind() == kind_t::inner); + IMMER_ASSERT_TAGGED(dst->kind() == kind_t::inner); + IMMER_ASSERT_TAGGED(src->kind() == kind_t::inner); auto src_r = src->relaxed(); auto dst_r = dst->relaxed(); inc_nodes(src->inner(), n); @@ -570,6 +609,23 @@ struct node return dst; } + static node_t* do_copy_inner_replace_r( + node_t* dst, node_t* src, count_t n, count_t offset, node_t* child) + { + IMMER_ASSERT_TAGGED(dst->kind() == kind_t::inner); + IMMER_ASSERT_TAGGED(src->kind() == kind_t::inner); + auto src_r = src->relaxed(); + auto dst_r = dst->relaxed(); + auto p = src->inner(); + inc_nodes(p, offset); + inc_nodes(p + offset + 1, n - offset - 1); + std::copy(src->inner(), src->inner() + n, dst->inner()); + std::copy(src_r->d.sizes, src_r->d.sizes + n, dst_r->d.sizes); + dst_r->d.count = n; + dst->inner()[offset] = child; + return dst; + } + static node_t* do_copy_inner_sr(node_t* dst, node_t* src, count_t n) { if (embed_relaxed) @@ -581,28 +637,47 @@ struct node } } + static node_t* do_copy_inner_replace_sr( + node_t* dst, node_t* src, count_t n, count_t offset, node_t* child) + { + if (embed_relaxed) + return do_copy_inner_replace_r(dst, src, n, offset, child); + else { + auto p = src->inner(); + inc_nodes(p, offset); + inc_nodes(p + offset + 1, n - offset - 1); + std::copy(p, p + n, dst->inner()); + dst->inner()[offset] = child; + return dst; + } + } + static node_t* copy_leaf(node_t* src, count_t n) { - assert(src->kind() == kind_t::leaf); + IMMER_ASSERT_TAGGED(src->kind() == kind_t::leaf); auto dst = make_leaf_n(n); - try { - std::uninitialized_copy(src->leaf(), src->leaf() + n, dst->leaf()); - } catch (...) { + IMMER_TRY { + detail::uninitialized_copy( + src->leaf(), src->leaf() + n, dst->leaf()); + } + IMMER_CATCH (...) { heap::deallocate(node_t::sizeof_leaf_n(n), dst); - throw; + IMMER_RETHROW; } return dst; } static node_t* copy_leaf_e(edit_t e, node_t* src, count_t n) { - assert(src->kind() == kind_t::leaf); + IMMER_ASSERT_TAGGED(src->kind() == kind_t::leaf); auto dst = make_leaf_e(e); - try { - std::uninitialized_copy(src->leaf(), src->leaf() + n, dst->leaf()); - } catch (...) { + IMMER_TRY { + detail::uninitialized_copy( + src->leaf(), src->leaf() + n, dst->leaf()); + } + IMMER_CATCH (...) { heap::deallocate(node_t::max_sizeof_leaf, dst); - throw; + IMMER_RETHROW; } return dst; } @@ -610,90 +685,96 @@ struct node static node_t* copy_leaf_n(count_t allocn, node_t* src, count_t n) { assert(allocn >= n); - assert(src->kind() == kind_t::leaf); + IMMER_ASSERT_TAGGED(src->kind() == kind_t::leaf); auto dst = make_leaf_n(allocn); - try { - std::uninitialized_copy(src->leaf(), src->leaf() + n, dst->leaf()); - } catch (...) { + IMMER_TRY { + detail::uninitialized_copy( + src->leaf(), src->leaf() + n, dst->leaf()); + } + IMMER_CATCH (...) { heap::deallocate(node_t::sizeof_leaf_n(allocn), dst); - throw; + IMMER_RETHROW; } return dst; } - static node_t* copy_leaf(node_t* src1, count_t n1, - node_t* src2, count_t n2) + static node_t* copy_leaf(node_t* src1, count_t n1, node_t* src2, count_t n2) { - assert(src1->kind() == kind_t::leaf); - assert(src2->kind() == kind_t::leaf); + IMMER_ASSERT_TAGGED(src1->kind() == kind_t::leaf); + IMMER_ASSERT_TAGGED(src2->kind() == kind_t::leaf); auto dst = make_leaf_n(n1 + n2); - try { - std::uninitialized_copy( + IMMER_TRY { + detail::uninitialized_copy( src1->leaf(), src1->leaf() + n1, dst->leaf()); - } catch (...) { + } + IMMER_CATCH (...) { heap::deallocate(node_t::sizeof_leaf_n(n1 + n2), dst); - throw; + IMMER_RETHROW; } - try { - std::uninitialized_copy( + IMMER_TRY { + detail::uninitialized_copy( src2->leaf(), src2->leaf() + n2, dst->leaf() + n1); - } catch (...) { - destroy_n(dst->leaf(), n1); + } + IMMER_CATCH (...) { + detail::destroy_n(dst->leaf(), n1); heap::deallocate(node_t::sizeof_leaf_n(n1 + n2), dst); - throw; + IMMER_RETHROW; } return dst; } - static node_t* copy_leaf_e(edit_t e, - node_t* src1, count_t n1, - node_t* src2, count_t n2) + static node_t* + copy_leaf_e(edit_t e, node_t* src1, count_t n1, node_t* src2, count_t n2) { - assert(src1->kind() == kind_t::leaf); - assert(src2->kind() == kind_t::leaf); + IMMER_ASSERT_TAGGED(src1->kind() == kind_t::leaf); + IMMER_ASSERT_TAGGED(src2->kind() == kind_t::leaf); auto dst = make_leaf_e(e); - try { - std::uninitialized_copy( + IMMER_TRY { + detail::uninitialized_copy( src1->leaf(), src1->leaf() + n1, dst->leaf()); - } catch (...) { + } + IMMER_CATCH (...) { heap::deallocate(max_sizeof_leaf, dst); - throw; + IMMER_RETHROW; } - try { - std::uninitialized_copy( + IMMER_TRY { + detail::uninitialized_copy( src2->leaf(), src2->leaf() + n2, dst->leaf() + n1); - } catch (...) { - destroy_n(dst->leaf(), n1); + } + IMMER_CATCH (...) { + detail::destroy_n(dst->leaf(), n1); heap::deallocate(max_sizeof_leaf, dst); - throw; + IMMER_RETHROW; } return dst; } static node_t* copy_leaf_e(edit_t e, node_t* src, count_t idx, count_t last) { - assert(src->kind() == kind_t::leaf); + IMMER_ASSERT_TAGGED(src->kind() == kind_t::leaf); auto dst = make_leaf_e(e); - try { - std::uninitialized_copy( + IMMER_TRY { + detail::uninitialized_copy( src->leaf() + idx, src->leaf() + last, dst->leaf()); - } catch (...) { + } + IMMER_CATCH (...) { heap::deallocate(max_sizeof_leaf, dst); - throw; + IMMER_RETHROW; } return dst; } static node_t* copy_leaf(node_t* src, count_t idx, count_t last) { - assert(src->kind() == kind_t::leaf); + IMMER_ASSERT_TAGGED(src->kind() == kind_t::leaf); auto dst = make_leaf_n(last - idx); - try { - std::uninitialized_copy( + IMMER_TRY { + detail::uninitialized_copy( src->leaf() + idx, src->leaf() + last, dst->leaf()); - } catch (...) { + } + IMMER_CATCH (...) { heap::deallocate(node_t::sizeof_leaf_n(last - idx), dst); - throw; + IMMER_RETHROW; } return dst; } @@ -702,28 +783,29 @@ struct node static node_t* copy_leaf_emplace(node_t* src, count_t n, U&& x) { auto dst = copy_leaf_n(n + 1, src, n); - try { + IMMER_TRY { new (dst->leaf() + n) T{std::forward(x)}; - } catch (...) { - destroy_n(dst->leaf(), n); + } + IMMER_CATCH (...) { + detail::destroy_n(dst->leaf(), n); heap::deallocate(node_t::sizeof_leaf_n(n + 1), dst); - throw; + IMMER_RETHROW; } return dst; } static void delete_inner(node_t* p, count_t n) { - assert(p->kind() == kind_t::inner); + IMMER_ASSERT_TAGGED(p->kind() == kind_t::inner); assert(!p->relaxed()); - heap::deallocate(ownee(p).owned() - ? node_t::max_sizeof_inner - : node_t::sizeof_inner_n(n), p); + heap::deallocate(ownee(p).owned() ? node_t::max_sizeof_inner + : node_t::sizeof_inner_n(n), + p); } static void delete_inner_e(node_t* p) { - assert(p->kind() == kind_t::inner); + IMMER_ASSERT_TAGGED(p->kind() == kind_t::inner); assert(!p->relaxed()); heap::deallocate(node_t::max_sizeof_inner, p); } @@ -738,26 +820,27 @@ struct node static void delete_inner_r(node_t* p, count_t n) { - assert(p->kind() == kind_t::inner); + IMMER_ASSERT_TAGGED(p->kind() == kind_t::inner); auto r = p->relaxed(); assert(r); - static_if([&] (auto) { + static_if([&](auto) { if (node_t::refs(r).dec()) heap::deallocate(node_t::ownee(r).owned() - ? node_t::max_sizeof_relaxed - : node_t::sizeof_relaxed_n(n), r); + ? node_t::max_sizeof_relaxed + : node_t::sizeof_relaxed_n(n), + r); }); - heap::deallocate(ownee(p).owned() - ? node_t::max_sizeof_inner_r - : node_t::sizeof_inner_r_n(n), p); + heap::deallocate(ownee(p).owned() ? node_t::max_sizeof_inner_r + : node_t::sizeof_inner_r_n(n), + p); } static void delete_inner_r_e(node_t* p) { - assert(p->kind() == kind_t::inner); + IMMER_ASSERT_TAGGED(p->kind() == kind_t::inner); auto r = p->relaxed(); assert(r); - static_if([&] (auto) { + static_if([&](auto) { if (node_t::refs(r).dec()) heap::deallocate(node_t::max_sizeof_relaxed, r); }); @@ -766,38 +849,40 @@ struct node static void delete_leaf(node_t* p, count_t n) { - assert(p->kind() == kind_t::leaf); - destroy_n(p->leaf(), n); - heap::deallocate(ownee(p).owned() - ? node_t::max_sizeof_leaf - : node_t::sizeof_leaf_n(n), p); + IMMER_ASSERT_TAGGED(p->kind() == kind_t::leaf); + detail::destroy_n(p->leaf(), n); + heap::deallocate(ownee(p).owned() ? node_t::max_sizeof_leaf + : node_t::sizeof_leaf_n(n), + p); } bool can_mutate(edit_t e) const { - return refs(this).unique() - || ownee(this).can_mutate(e); + return refs(this).unique() || ownee(this).can_mutate(e); } - bool can_relax() const - { - return !embed_relaxed || relaxed(); - } + bool can_relax() const { return !embed_relaxed || relaxed(); } relaxed_t* ensure_mutable_relaxed(edit_t e) { auto src_r = relaxed(); return static_if( - [&] (auto) { return src_r; }, - [&] (auto) { + [&](auto) { return src_r; }, + [&](auto) { if (node_t::refs(src_r).unique() || node_t::ownee(src_r).can_mutate(e)) return src_r; else { - if (src_r) - node_t::refs(src_r).dec_unsafe(); auto dst_r = impl.d.data.inner.relaxed = new (heap::allocate(max_sizeof_relaxed)) relaxed_t; + if (src_r) { + auto n = dst_r->d.count = src_r->d.count; + std::copy( + src_r->d.sizes, src_r->d.sizes + n, dst_r->d.sizes); + if (node_t::refs(src_r).dec()) + heap::deallocate(node_t::sizeof_inner_r_n(n), + src_r); + } node_t::ownee(dst_r) = e; return dst_r; } @@ -808,17 +893,23 @@ struct node { auto src_r = relaxed(); return static_if( - [&] (auto) { return src_r; }, - [&] (auto) { + [&](auto) { return src_r; }, + [&](auto) { if (src_r && (node_t::refs(src_r).unique() || node_t::ownee(src_r).can_mutate(e))) { node_t::ownee(src_r) = ec; return src_r; } else { - if (src_r) - node_t::refs(src_r).dec_unsafe(); auto dst_r = impl.d.data.inner.relaxed = new (heap::allocate(max_sizeof_relaxed)) relaxed_t; + if (src_r) { + auto n = dst_r->d.count = src_r->d.count; + std::copy( + src_r->d.sizes, src_r->d.sizes + n, dst_r->d.sizes); + if (node_t::refs(src_r).dec()) + heap::deallocate(node_t::sizeof_inner_r_n(n), + src_r); + } node_t::ownee(dst_r) = ec; return dst_r; } @@ -829,18 +920,23 @@ struct node { auto src_r = relaxed(); return static_if( - [&] (auto) { return src_r; }, - [&] (auto) { + [&](auto) { return src_r; }, + [&](auto) { if (node_t::refs(src_r).unique() || node_t::ownee(src_r).can_mutate(e)) return src_r; else { - if (src_r) - node_t::refs(src_r).dec_unsafe(); auto dst_r = new (heap::allocate(max_sizeof_relaxed)) relaxed_t; - std::copy(src_r->d.sizes, src_r->d.sizes + n, dst_r->d.sizes); - node_t::ownee(dst_r) = e; + if (src_r) { + std::copy( + src_r->d.sizes, src_r->d.sizes + n, dst_r->d.sizes); + if (node_t::refs(src_r).dec()) + heap::deallocate(node_t::sizeof_inner_r_n(n), + src_r); + } + dst_r->d.count = n; + node_t::ownee(dst_r) = e; return impl.d.data.inner.relaxed = dst_r; } }); @@ -859,7 +955,6 @@ struct node } bool dec() const { return refs(this).dec(); } - void dec_unsafe() const { refs(this).dec_unsafe(); } static void inc_nodes(node_t** p, count_t n) { @@ -867,13 +962,13 @@ struct node refs(*i).inc(); } -#if IMMER_RBTS_TAGGED_NODE +#if IMMER_TAGGED_NODE shift_t compute_shift() { if (kind() == kind_t::leaf) return endshift; else - return B + inner() [0]->compute_shift(); + return B + inner()[0]->compute_shift(); } #endif @@ -882,7 +977,7 @@ struct node #if IMMER_DEBUG_DEEP_CHECK assert(size > 0); if (shift == endshift) { - assert(kind() == kind_t::leaf); + IMMER_ASSERT_TAGGED(kind() == kind_t::leaf); assert(size <= branches); } else if (auto r = relaxed()) { auto count = r->d.count; @@ -894,28 +989,23 @@ struct node IMMER_TRACE_E(size); } assert(r->d.sizes[count - 1] == size); - for (auto i = 1; i < count; ++i) + for (auto i = 1u; i < count; ++i) assert(r->d.sizes[i - 1] < r->d.sizes[i]); auto last_size = size_t{}; - for (auto i = 0; i < count; ++i) { - assert(inner()[i]->check( - shift - B, - r->d.sizes[i] - last_size)); + for (auto i = 0u; i < count; ++i) { + assert(inner()[i]->check(shift - B, r->d.sizes[i] - last_size)); last_size = r->d.sizes[i]; } } else { assert(size <= branches << shift); - auto count = (size >> shift) - + (size - ((size >> shift) << shift) > 0); + auto count = + (size >> shift) + (size - ((size >> shift) << shift) > 0); assert(count <= branches); if (count) { - for (auto i = 1; i < count - 1; ++i) - assert(inner()[i]->check( - shift - B, - 1 << shift)); + for (auto i = 1u; i < count - 1; ++i) + assert(inner()[i]->check(shift - B, 1 << shift)); assert(inner()[count - 1]->check( - shift - B, - size - ((count - 1) << shift))); + shift - B, size - ((count - 1) << shift))); } } #endif // IMMER_DEBUG_DEEP_CHECK @@ -926,11 +1016,12 @@ struct node template constexpr bits_t derive_bits_leaf_aux() { - using node_t = node; + using node_t = node; constexpr auto sizeof_elem = sizeof(T); - constexpr auto space = node_t::max_sizeof_inner - node_t::sizeof_packed_leaf_n(0); + constexpr auto space = + node_t::max_sizeof_inner - node_t::sizeof_packed_leaf_n(0); constexpr auto full_elems = space / sizeof_elem; - constexpr auto BL = log2(full_elems); + constexpr auto BL = log2(full_elems); return BL; } diff --git a/src/immer/detail/rbts/operations.hpp b/src/immer/detail/rbts/operations.hpp index 17ec9911ef..d5ff67934b 100644 --- a/src/immer/detail/rbts/operations.hpp +++ b/src/immer/detail/rbts/operations.hpp @@ -14,10 +14,10 @@ #include #include -#include -#include #include #include +#include +#include namespace immer { namespace detail { @@ -30,26 +30,34 @@ struct array_for_visitor : visitor_base> template static T* visit_inner(PosT&& pos, size_t idx) - { return pos.descend(this_t{}, idx); } + { + return pos.descend(this_t{}, idx); + } template static T* visit_leaf(PosT&& pos, size_t) - { return pos.node()->leaf(); } + { + return pos.node()->leaf(); + } }; template struct region_for_visitor : visitor_base> { - using this_t = region_for_visitor; + using this_t = region_for_visitor; using result_t = std::tuple; template static result_t visit_inner(PosT&& pos, size_t idx) - { return pos.towards(this_t{}, idx); } + { + return pos.towards(this_t{}, idx); + } template static result_t visit_leaf(PosT&& pos, size_t idx) - { return { pos.node()->leaf(), pos.index(idx), pos.count() }; } + { + return std::make_tuple(pos.node()->leaf(), pos.index(idx), pos.count()); + } }; template @@ -59,11 +67,15 @@ struct get_visitor : visitor_base> template static const T& visit_inner(PosT&& pos, size_t idx) - { return pos.descend(this_t{}, idx); } + { + return pos.descend(this_t{}, idx); + } template static const T& visit_leaf(PosT&& pos, size_t idx) - { return pos.node()->leaf() [pos.index(idx)]; } + { + return pos.node()->leaf()[pos.index(idx)]; + } }; struct for_each_chunk_visitor : visitor_base @@ -72,13 +84,15 @@ struct for_each_chunk_visitor : visitor_base template static void visit_inner(Pos&& pos, Fn&& fn) - { pos.each(this_t{}, fn); } + { + pos.each(this_t{}, fn); + } template static void visit_leaf(Pos&& pos, Fn&& fn) { auto data = pos.node()->leaf(); - fn(data, data + pos.count()); + fn(as_const(data), as_const(data) + pos.count()); } }; @@ -88,12 +102,14 @@ struct for_each_chunk_p_visitor : visitor_base template static bool visit_inner(Pos&& pos, Fn&& fn) - { return pos.each_pred(this_t{}, fn); } + { + return pos.each_pred(this_t{}, fn); + } template static bool visit_leaf(Pos&& pos, Fn&& fn) { - auto data = pos.node()->leaf(); + auto data = as_const(pos.node()->leaf()); return fn(data, data + pos.count()); } }; @@ -103,8 +119,7 @@ struct for_each_chunk_left_visitor : visitor_base using this_t = for_each_chunk_left_visitor; template - static void visit_inner(Pos&& pos, - size_t last, Fn&& fn) + static void visit_inner(Pos&& pos, size_t last, Fn&& fn) { auto l = pos.index(last); pos.each_left(for_each_chunk_visitor{}, l, fn); @@ -112,12 +127,10 @@ struct for_each_chunk_left_visitor : visitor_base } template - static void visit_leaf(Pos&& pos, - size_t last, - Fn&& fn) + static void visit_leaf(Pos&& pos, size_t last, Fn&& fn) { auto data = pos.node()->leaf(); - auto l = pos.index(last); + auto l = pos.index(last); fn(data, data + l + 1); } }; @@ -127,8 +140,7 @@ struct for_each_chunk_right_visitor : visitor_base using this_t = for_each_chunk_right_visitor; template - static void visit_inner(Pos&& pos, - size_t first, Fn&& fn) + static void visit_inner(Pos&& pos, size_t first, Fn&& fn) { auto f = pos.index(first); pos.towards_oh(this_t{}, first, f, fn); @@ -136,12 +148,10 @@ struct for_each_chunk_right_visitor : visitor_base } template - static void visit_leaf(Pos&& pos, - size_t first, - Fn&& fn) + static void visit_leaf(Pos&& pos, size_t first, Fn&& fn) { auto data = pos.node()->leaf(); - auto f = pos.index(first); + auto f = pos.index(first); fn(data + f, data + pos.count()); } }; @@ -151,13 +161,11 @@ struct for_each_chunk_i_visitor : visitor_base using this_t = for_each_chunk_i_visitor; template - static void visit_relaxed(Pos&& pos, - size_t first, size_t last, - Fn&& fn) + static void visit_relaxed(Pos&& pos, size_t first, size_t last, Fn&& fn) { // we are going towards *two* indices, so we need to do the // relaxed as a special case to correct the second index - if (first < last) { + if (first < last) { auto f = pos.index(first); auto l = pos.index(last - 1); if (f == l) { @@ -173,11 +181,9 @@ struct for_each_chunk_i_visitor : visitor_base } template - static void visit_regular(Pos&& pos, - size_t first, size_t last, - Fn&& fn) + static void visit_regular(Pos&& pos, size_t first, size_t last, Fn&& fn) { - if (first < last) { + if (first < last) { auto f = pos.index(first); auto l = pos.index(last - 1); if (f == l) @@ -192,9 +198,7 @@ struct for_each_chunk_i_visitor : visitor_base } template - static void visit_leaf(Pos&& pos, - size_t first, size_t last, - Fn&& fn) + static void visit_leaf(Pos&& pos, size_t first, size_t last, Fn&& fn) { auto data = pos.node()->leaf(); if (first < last) { @@ -211,21 +215,18 @@ struct for_each_chunk_p_left_visitor using this_t = for_each_chunk_p_left_visitor; template - static bool visit_inner(Pos&& pos, - size_t last, Fn&& fn) + static bool visit_inner(Pos&& pos, size_t last, Fn&& fn) { auto l = pos.index(last); - return pos.each_pred_left(for_each_chunk_p_visitor{}, l, fn) - && pos.towards_oh(this_t{}, last, l, fn); + return pos.each_pred_left(for_each_chunk_p_visitor{}, l, fn) && + pos.towards_oh(this_t{}, last, l, fn); } template - static bool visit_leaf(Pos&& pos, - size_t last, - Fn&& fn) + static bool visit_leaf(Pos&& pos, size_t last, Fn&& fn) { auto data = pos.node()->leaf(); - auto l = pos.index(last); + auto l = pos.index(last); return fn(data, data + l + 1); } }; @@ -236,21 +237,18 @@ struct for_each_chunk_p_right_visitor using this_t = for_each_chunk_p_right_visitor; template - static bool visit_inner(Pos&& pos, - size_t first, Fn&& fn) + static bool visit_inner(Pos&& pos, size_t first, Fn&& fn) { auto f = pos.index(first); - return pos.towards_oh(this_t{}, first, f, fn) - && pos.each_pred_right(for_each_chunk_p_visitor{}, f + 1, fn); + return pos.towards_oh(this_t{}, first, f, fn) && + pos.each_pred_right(for_each_chunk_p_visitor{}, f + 1, fn); } template - static bool visit_leaf(Pos&& pos, - size_t first, - Fn&& fn) + static bool visit_leaf(Pos&& pos, size_t first, Fn&& fn) { auto data = pos.node()->leaf(); - auto f = pos.index(first); + auto f = pos.index(first); return fn(data + f, data + pos.count()); } }; @@ -260,52 +258,53 @@ struct for_each_chunk_p_i_visitor : visitor_base using this_t = for_each_chunk_p_i_visitor; template - static bool visit_relaxed(Pos&& pos, - size_t first, size_t last, - Fn&& fn) + static bool visit_relaxed(Pos&& pos, size_t first, size_t last, Fn&& fn) { // we are going towards *two* indices, so we need to do the // relaxed as a special case to correct the second index - if (first < last) { + if (first < last) { auto f = pos.index(first); auto l = pos.index(last - 1); if (f == l) { auto sbh = pos.size_before(f); - return pos.towards_oh_sbh(this_t{}, first, f, sbh, last - sbh, fn); + return pos.towards_oh_sbh( + this_t{}, first, f, sbh, last - sbh, fn); } else { assert(f < l); - return pos.towards_oh(for_each_chunk_p_right_visitor{}, first, f, fn) - && pos.each_pred_i(for_each_chunk_p_visitor{}, f + 1, l, fn) - && pos.towards_oh(for_each_chunk_p_left_visitor{}, last - 1, l, fn); + return pos.towards_oh( + for_each_chunk_p_right_visitor{}, first, f, fn) && + pos.each_pred_i( + for_each_chunk_p_visitor{}, f + 1, l, fn) && + pos.towards_oh( + for_each_chunk_p_left_visitor{}, last - 1, l, fn); } } return true; } template - static bool visit_regular(Pos&& pos, - size_t first, size_t last, - Fn&& fn) + static bool visit_regular(Pos&& pos, size_t first, size_t last, Fn&& fn) { - if (first < last) { + if (first < last) { auto f = pos.index(first); auto l = pos.index(last - 1); if (f == l) return pos.towards_oh(this_t{}, first, f, last, fn); else { assert(f < l); - return pos.towards_oh(for_each_chunk_p_right_visitor{}, first, f, fn) - && pos.each_pred_i(for_each_chunk_p_visitor{}, f + 1, l, fn) - && pos.towards_oh(for_each_chunk_p_left_visitor{}, last - 1, l, fn); + return pos.towards_oh( + for_each_chunk_p_right_visitor{}, first, f, fn) && + pos.each_pred_i( + for_each_chunk_p_visitor{}, f + 1, l, fn) && + pos.towards_oh( + for_each_chunk_p_left_visitor{}, last - 1, l, fn); } } return true; } template - static bool visit_leaf(Pos&& pos, - size_t first, size_t last, - Fn&& fn) + static bool visit_leaf(Pos&& pos, size_t first, size_t last, Fn&& fn) { auto data = pos.node()->leaf(); if (first < last) { @@ -323,37 +322,48 @@ struct equals_visitor : visitor_base struct this_aux_t : visitor_base { - template - static bool visit_inner(PosR&& posr, - count_t i, PosL&& posl, - Iter&& first, size_t idx) - { return posl.nth_sub(i, this_t{}, posr, first, idx); } - - template - static bool visit_leaf(PosR&& posr, - count_t i, PosL&& posl, - Iter&& first, size_t idx) - { return posl.nth_sub_leaf(i, this_t{}, posr, first, idx); } + template + static bool visit_inner( + PosR&& posr, count_t i, PosL&& posl, Iter&& first, size_t idx) + { + return posl.nth_sub(i, this_t{}, posr, first, idx); + } + + template + static bool visit_leaf( + PosR&& posr, count_t i, PosL&& posl, Iter&& first, size_t idx) + { + return posl.nth_sub_leaf(i, this_t{}, posr, first, idx); + } }; struct rrb : visitor_base { template - static bool visit_node(PosR&& posr, Iter&& first, - Node* rootl, shift_t shiftl, size_t sizel) + static bool visit_node(PosR&& posr, + Iter&& first, + Node* rootl, + shift_t shiftl, + size_t sizel) { assert(shiftl <= posr.shift()); return shiftl == posr.shift() - ? visit_maybe_relaxed_sub(rootl, shiftl, sizel, - this_t{}, posr, first, size_t{}) - : posr.first_sub_inner(rrb{}, first, rootl, shiftl, sizel); + ? visit_maybe_relaxed_sub(rootl, + shiftl, + sizel, + this_t{}, + posr, + first, + size_t{}) + : posr.first_sub_inner( + rrb{}, first, rootl, shiftl, sizel); } }; template static auto equal_chunk_p(Iter&& iter) { - return [iter] (auto f, auto e) mutable { + return [iter](auto f, auto e) mutable { if (f == &*iter) { iter += e - f; return true; @@ -366,8 +376,8 @@ struct equals_visitor : visitor_base } template - static bool visit_relaxed(PosL&& posl, PosR&& posr, - Iter&& first, size_t idx) + static bool + visit_relaxed(PosL&& posl, PosR&& posr, Iter&& first, size_t idx) { auto nl = posl.node(); auto nr = posr.node(); @@ -377,16 +387,20 @@ struct equals_visitor : visitor_base auto cr = posr.count(); assert(cr > 0); auto sbr = size_t{}; - auto i = count_t{}; - auto j = count_t{}; + auto i = count_t{}; + auto j = count_t{}; for (; i < cl; ++i) { auto sbl = posl.size_before(i); - for (; j + 1 < cr && (sbr = posr.size_before(j)) < sbl; ++j); - auto res = sbl == sbr - ? posr.nth_sub(j, this_aux_t{}, i, posl, first, idx + sbl) - : posl.nth_sub(i, for_each_chunk_p_visitor{}, - this_t::equal_chunk_p(first + (idx + sbl))); - if (!res) return false; + for (; j + 1 < cr && (sbr = posr.size_before(j)) < sbl; ++j) + ; + auto res = + sbl == sbr + ? posr.nth_sub(j, this_aux_t{}, i, posl, first, idx + sbl) + : posl.nth_sub(i, + for_each_chunk_p_visitor{}, + this_t::equal_chunk_p(first + (idx + sbl))); + if (!res) + return false; } return true; } @@ -403,43 +417,40 @@ struct equals_visitor : visitor_base visit_regular(PosL&& posl, PosR&& posr, Iter&& first, size_t idx) { return posl.count() >= posr.count() - ? this_t::visit_regular(posl, posr.node()) - : this_t::visit_regular(posr, posl.node()); + ? this_t::visit_regular(posl, posr.node()) + : this_t::visit_regular(posr, posl.node()); } template - static bool visit_leaf(PosL&& posl, - PosR&& posr, Iter&& first, size_t idx) + static bool visit_leaf(PosL&& posl, PosR&& posr, Iter&& first, size_t idx) { if (posl.node() == posr.node()) return true; auto cl = posl.count(); auto cr = posr.count(); auto mp = std::min(cl, cr); - return - std::equal(posl.node()->leaf(), - posl.node()->leaf() + mp, - posr.node()->leaf()) && - std::equal(posl.node()->leaf() + mp, - posl.node()->leaf() + posl.count(), - first + (idx + mp)); + return std::equal(posl.node()->leaf(), + posl.node()->leaf() + mp, + posr.node()->leaf()) && + std::equal(posl.node()->leaf() + mp, + posl.node()->leaf() + posl.count(), + first + (idx + mp)); } template static bool visit_regular(Pos&& pos, NodeT* other) { auto node = pos.node(); - return node == other - || pos.each_pred_zip(this_t{}, other); + return node == other || pos.each_pred_zip(this_t{}, other); } template static bool visit_leaf(Pos&& pos, NodeT* other) { auto node = pos.node(); - return node == other - || std::equal(node->leaf(), node->leaf() + pos.count(), - other->leaf()); + return node == other || std::equal(node->leaf(), + node->leaf() + pos.count(), + other->leaf()); } }; @@ -452,51 +463,52 @@ struct update_visitor : visitor_base> template static node_t* visit_relaxed(Pos&& pos, size_t idx, Fn&& fn) { - auto offset = pos.index(idx); - auto count = pos.count(); - auto node = node_t::make_inner_sr_n(count, pos.relaxed()); - try { + auto offset = pos.index(idx); + auto count = pos.count(); + auto node = node_t::make_inner_sr_n(count, pos.relaxed()); + IMMER_TRY { auto child = pos.towards_oh(this_t{}, idx, offset, fn); - node_t::do_copy_inner_sr(node, pos.node(), count); - node->inner()[offset]->dec_unsafe(); - node->inner()[offset] = child; + node_t::do_copy_inner_replace_sr( + node, pos.node(), count, offset, child); return node; - } catch (...) { + } + IMMER_CATCH (...) { node_t::delete_inner_r(node, count); - throw; + IMMER_RETHROW; } } template static node_t* visit_regular(Pos&& pos, size_t idx, Fn&& fn) { - auto offset = pos.index(idx); - auto count = pos.count(); - auto node = node_t::make_inner_n(count); - try { + auto offset = pos.index(idx); + auto count = pos.count(); + auto node = node_t::make_inner_n(count); + IMMER_TRY { auto child = pos.towards_oh_ch(this_t{}, idx, offset, count, fn); - node_t::do_copy_inner(node, pos.node(), count); - node->inner()[offset]->dec_unsafe(); - node->inner()[offset] = child; + node_t::do_copy_inner_replace( + node, pos.node(), count, offset, child); return node; - } catch (...) { + } + IMMER_CATCH (...) { node_t::delete_inner(node, count); - throw; + IMMER_RETHROW; } } template static node_t* visit_leaf(Pos&& pos, size_t idx, Fn&& fn) { - auto offset = pos.index(idx); - auto node = node_t::copy_leaf(pos.node(), pos.count()); - try { - node->leaf()[offset] = std::forward(fn) ( - std::move(node->leaf()[offset])); + auto offset = pos.index(idx); + auto node = node_t::copy_leaf(pos.node(), pos.count()); + IMMER_TRY { + node->leaf()[offset] = + std::forward(fn)(std::move(node->leaf()[offset])); return node; - } catch (...) { + } + IMMER_CATCH (...) { node_t::delete_leaf(node, pos.count()); - throw; + IMMER_RETHROW; } } }; @@ -509,7 +521,7 @@ struct dec_visitor : visitor_base static void visit_relaxed(Pos&& p) { using node_t = node_type; - auto node = p.node(); + auto node = p.node(); if (node->dec()) { p.each(this_t{}); node_t::delete_inner_r(node, p.count()); @@ -520,7 +532,7 @@ struct dec_visitor : visitor_base static void visit_regular(Pos&& p) { using node_t = node_type; - auto node = p.node(); + auto node = p.node(); if (node->dec()) { p.each(this_t{}); node_t::delete_inner(node, p.count()); @@ -531,7 +543,7 @@ struct dec_visitor : visitor_base static void visit_leaf(Pos&& p) { using node_t = node_type; - auto node = p.node(); + auto node = p.node(); if (node->dec()) { node_t::delete_leaf(node, p.count()); } @@ -577,69 +589,75 @@ struct get_mut_visitor : visitor_base> using edit_t = typename NodeT::edit_t; template - static value_t& visit_relaxed(Pos&& pos, size_t idx, - edit_t e, node_t** location) + static value_t& + visit_relaxed(Pos&& pos, size_t idx, edit_t e, node_t** location) { - auto offset = pos.index(idx); - auto count = pos.count(); - auto node = pos.node(); + auto offset = pos.index(idx); + auto count = pos.count(); + auto node = pos.node(); if (node->can_mutate(e)) { - return pos.towards_oh(this_t{}, idx, offset, - e, &node->inner()[offset]); + return pos.towards_oh( + this_t{}, idx, offset, e, &node->inner()[offset]); } else { auto new_node = node_t::copy_inner_sr_e(e, node, count); - try { - auto& res = pos.towards_oh(this_t{}, idx, offset, - e, &new_node->inner()[offset]); + IMMER_TRY { + auto& res = pos.towards_oh( + this_t{}, idx, offset, e, &new_node->inner()[offset]); pos.visit(dec_visitor{}); *location = new_node; return res; - } catch (...) { + } + IMMER_CATCH (...) { dec_relaxed(new_node, pos.shift()); - throw; + IMMER_RETHROW; } } } template - static value_t& visit_regular(Pos&& pos, size_t idx, - edit_t e, node_t** location) + static value_t& + visit_regular(Pos&& pos, size_t idx, edit_t e, node_t** location) { assert(pos.node() == *location); - auto offset = pos.index(idx); - auto count = pos.count(); - auto node = pos.node(); + auto offset = pos.index(idx); + auto count = pos.count(); + auto node = pos.node(); if (node->can_mutate(e)) { - return pos.towards_oh_ch(this_t{}, idx, offset, count, - e, &node->inner()[offset]); + return pos.towards_oh_ch( + this_t{}, idx, offset, count, e, &node->inner()[offset]); } else { auto new_node = node_t::copy_inner_e(e, node, count); - try { - auto& res = pos.towards_oh_ch(this_t{}, idx, offset, count, - e, &new_node->inner()[offset]); + IMMER_TRY { + auto& res = pos.towards_oh_ch(this_t{}, + idx, + offset, + count, + e, + &new_node->inner()[offset]); pos.visit(dec_visitor{}); *location = new_node; return res; - } catch (...) { + } + IMMER_CATCH (...) { dec_regular(new_node, pos.shift(), pos.size()); - throw; + IMMER_RETHROW; } } } template - static value_t& visit_leaf(Pos&& pos, size_t idx, - edit_t e, node_t** location) + static value_t& + visit_leaf(Pos&& pos, size_t idx, edit_t e, node_t** location) { assert(pos.node() == *location); auto node = pos.node(); if (node->can_mutate(e)) { - return node->leaf() [pos.index(idx)]; + return node->leaf()[pos.index(idx)]; } else { auto new_node = node_t::copy_leaf_e(e, pos.node(), pos.count()); pos.visit(dec_visitor{}); *location = new_node; - return new_node->leaf() [pos.index(idx)]; + return new_node->leaf()[pos.index(idx)]; } } }; @@ -651,29 +669,30 @@ struct push_tail_mut_visitor static constexpr auto B = NodeT::bits; static constexpr auto BL = NodeT::bits_leaf; - using this_t = push_tail_mut_visitor; + using this_t = push_tail_mut_visitor; using this_no_mut_t = push_tail_mut_visitor; - using node_t = NodeT; - using edit_t = typename NodeT::edit_t; + using node_t = NodeT; + using edit_t = typename NodeT::edit_t; template static node_t* visit_relaxed(Pos&& pos, edit_t e, node_t* tail, count_t ts) { - auto node = pos.node(); - auto level = pos.shift(); - auto idx = pos.count() - 1; - auto children = pos.size(idx); - auto new_idx = children == size_t{1} << level || level == BL - ? idx + 1 : idx; + auto node = pos.node(); + auto level = pos.shift(); + auto idx = pos.count() - 1; + auto children = pos.size(idx); + auto new_idx = + children == size_t{1} << level || level == BL ? idx + 1 : idx; auto new_child = static_cast(nullptr); - auto mutate = Mutating && node->can_mutate(e); + auto mutate = Mutating && node->can_mutate(e); if (new_idx >= branches) return nullptr; else if (idx == new_idx) { - new_child = mutate - ? pos.last_oh_csh(this_t{}, idx, children, e, tail, ts) - : pos.last_oh_csh(this_no_mut_t{}, idx, children, e, tail, ts); + new_child = + mutate ? pos.last_oh_csh(this_t{}, idx, children, e, tail, ts) + : pos.last_oh_csh( + this_no_mut_t{}, idx, children, e, tail, ts); if (!new_child) { if (++new_idx < branches) new_child = node_t::make_path_e(e, level - B, tail); @@ -684,30 +703,34 @@ struct push_tail_mut_visitor new_child = node_t::make_path_e(e, level - B, tail); if (mutate) { - auto count = new_idx + 1; - auto relaxed = node->ensure_mutable_relaxed_n(e, new_idx); - node->inner()[new_idx] = new_child; + auto count = new_idx + 1; + auto relaxed = node->ensure_mutable_relaxed_n(e, new_idx); + node->inner()[new_idx] = new_child; relaxed->d.sizes[new_idx] = pos.size() + ts; - relaxed->d.count = count; + relaxed->d.count = count; + assert(relaxed->d.sizes[new_idx]); return node; } else { - try { + IMMER_TRY { auto count = new_idx + 1; auto new_node = node_t::copy_inner_r_e(e, pos.node(), new_idx); auto relaxed = new_node->relaxed(); new_node->inner()[new_idx] = new_child; - relaxed->d.sizes[new_idx] = pos.size() + ts; - relaxed->d.count = count; - if (Mutating) pos.visit(dec_visitor{}); + relaxed->d.sizes[new_idx] = pos.size() + ts; + relaxed->d.count = count; + assert(relaxed->d.sizes[new_idx]); + if (Mutating) + pos.visit(dec_visitor{}); return new_node; - } catch (...) { + } + IMMER_CATCH (...) { auto shift = pos.shift(); auto size = new_idx == idx ? children + ts : ts; if (shift > BL) { tail->inc(); dec_inner(new_child, shift - B, size); } - throw; + IMMER_RETHROW; } } } @@ -716,34 +739,41 @@ struct push_tail_mut_visitor static node_t* visit_regular(Pos&& pos, edit_t e, node_t* tail, Args&&...) { assert((pos.size() & mask) == 0); - auto node = pos.node(); - auto idx = pos.index(pos.size() - 1); - auto new_idx = pos.index(pos.size() + branches - 1); - auto mutate = Mutating && node->can_mutate(e); + auto node = pos.node(); + auto idx = pos.index(pos.size() - 1); + auto new_idx = pos.index(pos.size() + branches - 1); + auto mutate = Mutating && node->can_mutate(e); if (mutate) { node->inner()[new_idx] = - idx == new_idx ? pos.last_oh(this_t{}, idx, e, tail) - /* otherwise */ : node_t::make_path_e(e, pos.shift() - B, tail); + idx == new_idx ? pos.last_oh(this_t{}, idx, e, tail) + /* otherwise */ + : node_t::make_path_e(e, pos.shift() - B, tail); return node; } else { - auto new_parent = node_t::make_inner_e(e); - try { + auto new_parent = node_t::make_inner_e(e); + IMMER_TRY { new_parent->inner()[new_idx] = - idx == new_idx ? pos.last_oh(this_no_mut_t{}, idx, e, tail) - /* otherwise */ : node_t::make_path_e(e, pos.shift() - B, tail); + idx == new_idx + ? pos.last_oh(this_no_mut_t{}, idx, e, tail) + /* otherwise */ + : node_t::make_path_e(e, pos.shift() - B, tail); node_t::do_copy_inner(new_parent, node, new_idx); - if (Mutating) pos.visit(dec_visitor{}); + if (Mutating) + pos.visit(dec_visitor{}); return new_parent; - } catch (...) { + } + IMMER_CATCH (...) { node_t::delete_inner_e(new_parent); - throw; + IMMER_RETHROW; } } } template static node_t* visit_leaf(Pos&& pos, edit_t e, node_t* tail, Args&&...) - { IMMER_UNREACHABLE; } + { + IMMER_UNREACHABLE; + } }; template @@ -758,11 +788,11 @@ struct push_tail_visitor : visitor_base> template static node_t* visit_relaxed(Pos&& pos, node_t* tail, count_t ts) { - auto level = pos.shift(); - auto idx = pos.count() - 1; - auto children = pos.size(idx); - auto new_idx = children == size_t{1} << level || level == BL - ? idx + 1 : idx; + auto level = pos.shift(); + auto idx = pos.count() - 1; + auto children = pos.size(idx); + auto new_idx = + children == size_t{1} << level || level == BL ? idx + 1 : idx; auto new_child = static_cast(nullptr); if (new_idx >= branches) return nullptr; @@ -776,22 +806,25 @@ struct push_tail_visitor : visitor_base> } } else new_child = node_t::make_path(level - B, tail); - try { - auto count = new_idx + 1; - auto new_parent = node_t::copy_inner_r_n(count, pos.node(), new_idx); - auto new_relaxed = new_parent->relaxed(); - new_parent->inner()[new_idx] = new_child; + IMMER_TRY { + auto count = new_idx + 1; + auto new_parent = + node_t::copy_inner_r_n(count, pos.node(), new_idx); + auto new_relaxed = new_parent->relaxed(); + new_parent->inner()[new_idx] = new_child; new_relaxed->d.sizes[new_idx] = pos.size() + ts; - new_relaxed->d.count = count; + new_relaxed->d.count = count; + assert(new_relaxed->d.sizes[new_idx]); return new_parent; - } catch (...) { + } + IMMER_CATCH (...) { auto shift = pos.shift(); auto size = new_idx == idx ? children + ts : ts; if (shift > BL) { tail->inc(); dec_inner(new_child, shift - B, size); } - throw; + IMMER_RETHROW; } } @@ -799,24 +832,28 @@ struct push_tail_visitor : visitor_base> static node_t* visit_regular(Pos&& pos, node_t* tail, Args&&...) { assert((pos.size() & mask) == 0); - auto idx = pos.index(pos.size() - 1); - auto new_idx = pos.index(pos.size() + branches - 1); - auto count = new_idx + 1; - auto new_parent = node_t::make_inner_n(count); - try { + auto idx = pos.index(pos.size() - 1); + auto new_idx = pos.index(pos.size() + branches - 1); + auto count = new_idx + 1; + auto new_parent = node_t::make_inner_n(count); + IMMER_TRY { new_parent->inner()[new_idx] = - idx == new_idx ? pos.last_oh(this_t{}, idx, tail) - /* otherwise */ : node_t::make_path(pos.shift() - B, tail); - } catch (...) { + idx == new_idx ? pos.last_oh(this_t{}, idx, tail) + /* otherwise */ + : node_t::make_path(pos.shift() - B, tail); + } + IMMER_CATCH (...) { node_t::delete_inner(new_parent, count); - throw; + IMMER_RETHROW; } return node_t::do_copy_inner(new_parent, pos.node(), new_idx); } template static node_t* visit_leaf(Pos&& pos, node_t* tail, Args&&...) - { IMMER_UNREACHABLE; } + { + IMMER_UNREACHABLE; + } }; struct dec_right_visitor : visitor_base @@ -828,7 +865,7 @@ struct dec_right_visitor : visitor_base static void visit_relaxed(Pos&& p, count_t idx) { using node_t = node_type; - auto node = p.node(); + auto node = p.node(); if (node->dec()) { p.each_right(dec_t{}, idx); node_t::delete_inner_r(node, p.count()); @@ -839,7 +876,7 @@ struct dec_right_visitor : visitor_base static void visit_regular(Pos&& p, count_t idx) { using node_t = node_type; - auto node = p.node(); + auto node = p.node(); if (node->dec()) { p.each_right(dec_t{}, idx); node_t::delete_inner(node, p.count()); @@ -848,10 +885,12 @@ struct dec_right_visitor : visitor_base template static void visit_leaf(Pos&& p, count_t idx) - { IMMER_UNREACHABLE; } + { + IMMER_UNREACHABLE; + } }; -template +template struct slice_right_mut_visitor : visitor_base> { @@ -860,8 +899,8 @@ struct slice_right_mut_visitor using edit_t = typename NodeT::edit_t; // returns a new shift, new root, the new tail size and the new tail - using result_t = std::tuple; - using no_collapse_t = slice_right_mut_visitor; + using result_t = std::tuple; + using no_collapse_t = slice_right_mut_visitor; using no_collapse_no_mut_t = slice_right_mut_visitor; using no_mut_t = slice_right_mut_visitor; @@ -871,68 +910,77 @@ struct slice_right_mut_visitor template static result_t visit_relaxed(PosT&& pos, size_t last, edit_t e) { - auto idx = pos.index(last); - auto node = pos.node(); + auto idx = pos.index(last); + auto node = pos.node(); auto mutate = Mutating && node->can_mutate(e); if (Collapse && idx == 0) { - auto res = mutate - ? pos.towards_oh(this_t{}, last, idx, e) - : pos.towards_oh(no_mut_t{}, last, idx, e); - if (Mutating) pos.visit(dec_right_visitor{}, count_t{1}); + auto res = mutate ? pos.towards_oh(this_t{}, last, idx, e) + : pos.towards_oh(no_mut_t{}, last, idx, e); + if (Mutating) + pos.visit(dec_right_visitor{}, count_t{1}); return res; } else { using std::get; - auto subs = mutate - ? pos.towards_oh(no_collapse_t{}, last, idx, e) - : pos.towards_oh(no_collapse_no_mut_t{}, last, idx, e); + auto subs = + mutate ? pos.towards_oh(no_collapse_t{}, last, idx, e) + : pos.towards_oh(no_collapse_no_mut_t{}, last, idx, e); auto next = get<1>(subs); auto ts = get<2>(subs); auto tail = get<3>(subs); - try { + IMMER_TRY { if (next) { if (mutate) { auto nodr = node->ensure_mutable_relaxed_n(e, idx); pos.each_right(dec_visitor{}, idx + 1); node->inner()[idx] = next; nodr->d.sizes[idx] = last + 1 - ts; - nodr->d.count = idx + 1; - return { pos.shift(), node, ts, tail }; + nodr->d.count = idx + 1; + assert(nodr->d.sizes[idx]); + return std::make_tuple(pos.shift(), node, ts, tail); } else { auto newn = node_t::copy_inner_r_e(e, node, idx); auto newr = newn->relaxed(); newn->inner()[idx] = next; newr->d.sizes[idx] = last + 1 - ts; - newr->d.count = idx + 1; - if (Mutating) pos.visit(dec_visitor{}); - return { pos.shift(), newn, ts, tail }; + newr->d.count = idx + 1; + assert(newr->d.sizes[idx]); + if (Mutating) + pos.visit(dec_visitor{}); + return std::make_tuple(pos.shift(), newn, ts, tail); } } else if (idx == 0) { - if (Mutating) pos.visit(dec_right_visitor{}, count_t{1}); - return { pos.shift(), nullptr, ts, tail }; + if (Mutating) + pos.visit(dec_right_visitor{}, count_t{1}); + return std::make_tuple(pos.shift(), nullptr, ts, tail); } else if (Collapse && idx == 1 && pos.shift() > BL) { auto newn = pos.node()->inner()[0]; - if (!mutate) newn->inc(); - if (Mutating) pos.visit(dec_right_visitor{}, count_t{2}); - return { pos.shift() - B, newn, ts, tail }; + if (!mutate) + newn->inc(); + if (Mutating) + pos.visit(dec_right_visitor{}, count_t{2}); + return std::make_tuple(pos.shift() - B, newn, ts, tail); } else { if (mutate) { pos.each_right(dec_visitor{}, idx + 1); node->ensure_mutable_relaxed_n(e, idx)->d.count = idx; - return { pos.shift(), node, ts, tail }; + return std::make_tuple(pos.shift(), node, ts, tail); } else { auto newn = node_t::copy_inner_r_e(e, node, idx); - if (Mutating) pos.visit(dec_visitor{}); - return { pos.shift(), newn, ts, tail }; + if (Mutating) + pos.visit(dec_visitor{}); + return std::make_tuple(pos.shift(), newn, ts, tail); } } - } catch (...) { + } + IMMER_CATCH (...) { assert(!mutate); assert(!next || pos.shift() > BL); if (next) - dec_inner(next, pos.shift() - B, + dec_inner(next, + pos.shift() - B, last + 1 - ts - pos.size_before(idx)); dec_leaf(tail, ts); - throw; + IMMER_RETHROW; } } } @@ -940,60 +988,67 @@ struct slice_right_mut_visitor template static result_t visit_regular(PosT&& pos, size_t last, edit_t e) { - auto idx = pos.index(last); - auto node = pos.node(); + auto idx = pos.index(last); + auto node = pos.node(); auto mutate = Mutating && node->can_mutate(e); if (Collapse && idx == 0) { - auto res = mutate - ? pos.towards_oh(this_t{}, last, idx, e) - : pos.towards_oh(no_mut_t{}, last, idx, e); - if (Mutating) pos.visit(dec_right_visitor{}, count_t{1}); + auto res = mutate ? pos.towards_oh(this_t{}, last, idx, e) + : pos.towards_oh(no_mut_t{}, last, idx, e); + if (Mutating) + pos.visit(dec_right_visitor{}, count_t{1}); return res; } else { using std::get; - auto subs = mutate - ? pos.towards_oh(no_collapse_t{}, last, idx, e) - : pos.towards_oh(no_collapse_no_mut_t{}, last, idx, e); + auto subs = + mutate ? pos.towards_oh(no_collapse_t{}, last, idx, e) + : pos.towards_oh(no_collapse_no_mut_t{}, last, idx, e); auto next = get<1>(subs); auto ts = get<2>(subs); auto tail = get<3>(subs); - try { + IMMER_TRY { if (next) { if (mutate) { node->inner()[idx] = next; pos.each_right(dec_visitor{}, idx + 1); - return { pos.shift(), node, ts, tail }; + return std::make_tuple(pos.shift(), node, ts, tail); } else { - auto newn = node_t::copy_inner_e(e, node, idx); + auto newn = node_t::copy_inner_e(e, node, idx); newn->inner()[idx] = next; - if (Mutating) pos.visit(dec_visitor{}); - return { pos.shift(), newn, ts, tail }; + if (Mutating) + pos.visit(dec_visitor{}); + return std::make_tuple(pos.shift(), newn, ts, tail); } } else if (idx == 0) { - if (Mutating) pos.visit(dec_right_visitor{}, count_t{1}); - return { pos.shift(), nullptr, ts, tail }; + if (Mutating) + pos.visit(dec_right_visitor{}, count_t{1}); + return std::make_tuple(pos.shift(), nullptr, ts, tail); } else if (Collapse && idx == 1 && pos.shift() > BL) { auto newn = pos.node()->inner()[0]; - if (!mutate) newn->inc(); - if (Mutating) pos.visit(dec_right_visitor{}, count_t{2}); - return { pos.shift() - B, newn, ts, tail }; + if (!mutate) + newn->inc(); + if (Mutating) + pos.visit(dec_right_visitor{}, count_t{2}); + return std::make_tuple(pos.shift() - B, newn, ts, tail); } else { if (mutate) { pos.each_right(dec_visitor{}, idx + 1); - return { pos.shift(), node, ts, tail }; + return std::make_tuple(pos.shift(), node, ts, tail); } else { auto newn = node_t::copy_inner_e(e, node, idx); - if (Mutating) pos.visit(dec_visitor{}); - return { pos.shift(), newn, ts, tail }; + if (Mutating) + pos.visit(dec_visitor{}); + return std::make_tuple(pos.shift(), newn, ts, tail); } } - } catch (...) { + } + IMMER_CATCH (...) { assert(!mutate); assert(!next || pos.shift() > BL); assert(tail); - if (next) dec_regular(next, pos.shift() - B, last + 1 - ts); + if (next) + dec_regular(next, pos.shift() - B, last + 1 - ts); dec_leaf(tail, ts); - throw; + IMMER_RETHROW; } } } @@ -1006,28 +1061,30 @@ struct slice_right_mut_visitor auto node = pos.node(); auto mutate = Mutating && node->can_mutate(e); if (new_tail_size == old_tail_size) { - if (!Mutating) node->inc(); - return { 0, nullptr, new_tail_size, node }; + if (!Mutating) + node->inc(); + return std::make_tuple(0, nullptr, new_tail_size, node); } else if (mutate) { - destroy_n(node->leaf() + new_tail_size, - old_tail_size - new_tail_size); - return { 0, nullptr, new_tail_size, node }; + detail::destroy_n(node->leaf() + new_tail_size, + old_tail_size - new_tail_size); + return std::make_tuple(0, nullptr, new_tail_size, node); } else { auto new_tail = node_t::copy_leaf_e(e, node, new_tail_size); - if (Mutating) pos.visit(dec_visitor{}); - return { 0, nullptr, new_tail_size, new_tail }; + if (Mutating) + pos.visit(dec_visitor{}); + return std::make_tuple(0, nullptr, new_tail_size, new_tail); } } }; -template +template struct slice_right_visitor : visitor_base> { using node_t = NodeT; using this_t = slice_right_visitor; // returns a new shift, new root, the new tail size and the new tail - using result_t = std::tuple; + using result_t = std::tuple; using no_collapse_t = slice_right_visitor; static constexpr auto B = NodeT::bits; @@ -1045,30 +1102,36 @@ struct slice_right_visitor : visitor_base> auto next = get<1>(subs); auto ts = get<2>(subs); auto tail = get<3>(subs); - try { + IMMER_TRY { if (next) { auto count = idx + 1; auto newn = node_t::copy_inner_r_n(count, pos.node(), idx); auto newr = newn->relaxed(); newn->inner()[idx] = next; newr->d.sizes[idx] = last + 1 - ts; - newr->d.count = count; - return { pos.shift(), newn, ts, tail }; + newr->d.count = count; + assert(newr->d.sizes[idx]); + return std::make_tuple(pos.shift(), newn, ts, tail); } else if (idx == 0) { - return { pos.shift(), nullptr, ts, tail }; + return std::make_tuple(pos.shift(), nullptr, ts, tail); } else if (Collapse && idx == 1 && pos.shift() > BL) { auto newn = pos.node()->inner()[0]; - return { pos.shift() - B, newn->inc(), ts, tail }; + return std::make_tuple( + pos.shift() - B, newn->inc(), ts, tail); } else { auto newn = node_t::copy_inner_r(pos.node(), idx); - return { pos.shift(), newn, ts, tail }; + return std::make_tuple(pos.shift(), newn, ts, tail); } - } catch (...) { + } + IMMER_CATCH (...) { assert(!next || pos.shift() > BL); - if (next) dec_inner(next, pos.shift() - B, - last + 1 - ts - pos.size_before(idx)); - if (tail) dec_leaf(tail, ts); - throw; + if (next) + dec_inner(next, + pos.shift() - B, + last + 1 - ts - pos.size_before(idx)); + if (tail) + dec_leaf(tail, ts); + IMMER_RETHROW; } } } @@ -1085,26 +1148,29 @@ struct slice_right_visitor : visitor_base> auto next = get<1>(subs); auto ts = get<2>(subs); auto tail = get<3>(subs); - try { + IMMER_TRY { if (next) { - auto newn = node_t::copy_inner_n(idx + 1, pos.node(), idx); + auto newn = node_t::copy_inner_n(idx + 1, pos.node(), idx); newn->inner()[idx] = next; - return { pos.shift(), newn, ts, tail }; + return std::make_tuple(pos.shift(), newn, ts, tail); } else if (idx == 0) { - return { pos.shift(), nullptr, ts, tail }; + return std::make_tuple(pos.shift(), nullptr, ts, tail); } else if (Collapse && idx == 1 && pos.shift() > BL) { auto newn = pos.node()->inner()[0]; - return { pos.shift() - B, newn->inc(), ts, tail }; + return std::make_tuple( + pos.shift() - B, newn->inc(), ts, tail); } else { auto newn = node_t::copy_inner_n(idx, pos.node(), idx); - return { pos.shift(), newn, ts, tail }; + return std::make_tuple(pos.shift(), newn, ts, tail); } - } catch (...) { + } + IMMER_CATCH (...) { assert(!next || pos.shift() > BL); assert(tail); - if (next) dec_regular(next, pos.shift() - B, last + 1 - ts); + if (next) + dec_regular(next, pos.shift() - B, last + 1 - ts); dec_leaf(tail, ts); - throw; + IMMER_RETHROW; } } } @@ -1115,9 +1181,9 @@ struct slice_right_visitor : visitor_base> auto old_tail_size = pos.count(); auto new_tail_size = pos.index(last) + 1; auto new_tail = new_tail_size == old_tail_size - ? pos.node()->inc() - : node_t::copy_leaf(pos.node(), new_tail_size); - return { 0, nullptr, new_tail_size, new_tail }; + ? pos.node()->inc() + : node_t::copy_leaf(pos.node(), new_tail_size); + return std::make_tuple(0, nullptr, new_tail_size, new_tail); } }; @@ -1130,7 +1196,7 @@ struct dec_left_visitor : visitor_base static void visit_relaxed(Pos&& p, count_t idx) { using node_t = node_type; - auto node = p.node(); + auto node = p.node(); if (node->dec()) { p.each_left(dec_t{}, idx); node_t::delete_inner_r(node, p.count()); @@ -1141,7 +1207,7 @@ struct dec_left_visitor : visitor_base static void visit_regular(Pos&& p, count_t idx) { using node_t = node_type; - auto node = p.node(); + auto node = p.node(); if (node->dec()) { p.each_left(dec_t{}, idx); node_t::delete_inner(node, p.count()); @@ -1150,24 +1216,26 @@ struct dec_left_visitor : visitor_base template static void visit_leaf(Pos&& p, count_t idx) - { IMMER_UNREACHABLE; } + { + IMMER_UNREACHABLE; + } }; -template +template struct slice_left_mut_visitor : visitor_base> { - using node_t = NodeT; - using this_t = slice_left_mut_visitor; - using edit_t = typename NodeT::edit_t; - using value_t = typename NodeT::value_t; + using node_t = NodeT; + using this_t = slice_left_mut_visitor; + using edit_t = typename NodeT::edit_t; + using value_t = typename NodeT::value_t; using relaxed_t = typename NodeT::relaxed_t; // returns a new shift and new root using result_t = std::tuple; - using no_collapse_t = slice_left_mut_visitor; + using no_collapse_t = slice_left_mut_visitor; using no_collapse_no_mut_t = slice_left_mut_visitor; - using no_mut_t = slice_left_mut_visitor; + using no_mut_t = slice_left_mut_visitor; static constexpr auto B = NodeT::bits; static constexpr auto BL = NodeT::bits_leaf; @@ -1175,49 +1243,56 @@ struct slice_left_mut_visitor template static result_t visit_relaxed(PosT&& pos, size_t first, edit_t e) { - auto idx = pos.subindex(first); - auto count = pos.count(); - auto node = pos.node(); - auto mutate = Mutating && node->can_mutate(e); - auto left_size = pos.size_before(idx); - auto child_size = pos.size_sbh(idx, left_size); - auto dropped_size = first; + auto idx = pos.subindex(first); + auto count = pos.count(); + auto node = pos.node(); + auto mutate = Mutating && node->can_mutate(e); + auto left_size = pos.size_before(idx); + auto child_size = pos.size_sbh(idx, left_size); + auto dropped_size = first; auto child_dropped_size = dropped_size - left_size; if (Collapse && pos.shift() > BL && idx == pos.count() - 1) { - auto r = mutate - ? pos.towards_sub_oh(this_t{}, first, idx, e) - : pos.towards_sub_oh(no_mut_t{}, first, idx, e); - if (Mutating) pos.visit(dec_left_visitor{}, idx); + auto r = mutate ? pos.towards_sub_oh(this_t{}, first, idx, e) + : pos.towards_sub_oh(no_mut_t{}, first, idx, e); + if (mutate) + pos.visit(dec_left_visitor{}, idx); + else if (Mutating) + pos.visit(dec_visitor{}); return r; } else { using std::get; - auto newn = mutate - ? (node->ensure_mutable_relaxed(e), node) - : node_t::make_inner_r_e(e); - auto newr = newn->relaxed(); + auto newn = mutate ? (node->ensure_mutable_relaxed(e), node) + : node_t::make_inner_r_e(e); + auto newr = newn->relaxed(); auto newcount = count - idx; auto new_child_size = child_size - child_dropped_size; - try { - auto subs = mutate - ? pos.towards_sub_oh(no_collapse_t{}, first, idx, e) - : pos.towards_sub_oh(no_collapse_no_mut_t{}, first, idx, e); - if (mutate) pos.each_left(dec_visitor{}, idx); - pos.copy_sizes(idx + 1, newcount - 1, - new_child_size, newr->d.sizes + 1); - std::uninitialized_copy(node->inner() + idx + 1, - node->inner() + count, - newn->inner() + 1); + IMMER_TRY { + auto subs = + mutate ? pos.towards_sub_oh(no_collapse_t{}, first, idx, e) + : pos.towards_sub_oh( + no_collapse_no_mut_t{}, first, idx, e); + if (mutate) + pos.each_left(dec_visitor{}, idx); + pos.copy_sizes( + idx + 1, newcount - 1, new_child_size, newr->d.sizes + 1); + std::copy(node->inner() + idx + 1, + node->inner() + count, + newn->inner() + 1); newn->inner()[0] = get<1>(subs); newr->d.sizes[0] = new_child_size; - newr->d.count = newcount; + newr->d.count = newcount; + assert(new_child_size); if (!mutate) { node_t::inc_nodes(newn->inner() + 1, newcount - 1); - if (Mutating) pos.visit(dec_visitor{}); + if (Mutating) + pos.visit(dec_visitor{}); } - return { pos.shift(), newn }; - } catch (...) { - if (!mutate) node_t::delete_inner_r_e(newn); - throw; + return std::make_tuple(pos.shift(), newn); + } + IMMER_CATCH (...) { + if (!mutate) + node_t::delete_inner_r_e(newn); + IMMER_RETHROW; } } } @@ -1229,56 +1304,61 @@ struct slice_left_mut_visitor auto count = pos.count(); auto node = pos.node(); auto mutate = Mutating - // this is more restrictive than actually needed because - // it causes the algorithm to also avoid mutating the leaf - // in place - && !node_t::embed_relaxed - && node->can_mutate(e); - auto left_size = pos.size_before(idx); - auto child_size = pos.size_sbh(idx, left_size); - auto dropped_size = first; + // this is more restrictive than actually needed because + // it causes the algorithm to also avoid mutating the leaf + // in place + && !node_t::embed_relaxed && node->can_mutate(e); + auto left_size = pos.size_before(idx); + auto child_size = pos.size_sbh(idx, left_size); + auto dropped_size = first; auto child_dropped_size = dropped_size - left_size; if (Collapse && pos.shift() > BL && idx == pos.count() - 1) { - auto r = mutate - ? pos.towards_sub_oh(this_t{}, first, idx, e) - : pos.towards_sub_oh(no_mut_t{}, first, idx, e); - if (Mutating) pos.visit(dec_left_visitor{}, idx); + auto r = mutate ? pos.towards_sub_oh(this_t{}, first, idx, e) + : pos.towards_sub_oh(no_mut_t{}, first, idx, e); + if (mutate) + pos.visit(dec_left_visitor{}, idx); + else if (Mutating) + pos.visit(dec_visitor{}); return r; } else { using std::get; - // if possible, we convert the node to a relaxed one - // simply by allocating a `relaxed_t` size table for - // it... maybe some of this magic should be moved as a - // `node<...>` static method... + // if possible, we convert the node to a relaxed one simply by + // allocating a `relaxed_t` size table for it... maybe some of this + // magic should be moved as a `node<...>` static method... auto newcount = count - idx; - auto newn = mutate - ? (node->impl.d.data.inner.relaxed = new ( - node_t::heap::allocate( - node_t::max_sizeof_relaxed, - norefs_tag{})) relaxed_t, - node) - : node_t::make_inner_r_e(e); + auto newn = + mutate ? (node->impl.d.data.inner.relaxed = new ( + node_t::heap::allocate(node_t::max_sizeof_relaxed, + norefs_tag{})) relaxed_t, + node) + : node_t::make_inner_r_e(e); auto newr = newn->relaxed(); - try { - auto subs = mutate - ? pos.towards_sub_oh(no_collapse_t{}, first, idx, e) - : pos.towards_sub_oh(no_collapse_no_mut_t{}, first, idx, e); - if (mutate) pos.each_left(dec_visitor{}, idx); + IMMER_TRY { + auto subs = + mutate ? pos.towards_sub_oh(no_collapse_t{}, first, idx, e) + : pos.towards_sub_oh( + no_collapse_no_mut_t{}, first, idx, e); + if (mutate) + pos.each_left(dec_visitor{}, idx); newr->d.sizes[0] = child_size - child_dropped_size; - pos.copy_sizes(idx + 1, newcount - 1, - newr->d.sizes[0], newr->d.sizes + 1); - newr->d.count = newcount; + assert(newr->d.sizes[0]); + pos.copy_sizes( + idx + 1, newcount - 1, newr->d.sizes[0], newr->d.sizes + 1); + newr->d.count = newcount; newn->inner()[0] = get<1>(subs); - std::uninitialized_copy(node->inner() + idx + 1, - node->inner() + count, - newn->inner() + 1); + std::copy(node->inner() + idx + 1, + node->inner() + count, + newn->inner() + 1); if (!mutate) { node_t::inc_nodes(newn->inner() + 1, newcount - 1); - if (Mutating) pos.visit(dec_visitor{}); + if (Mutating) + pos.visit(dec_visitor{}); } - return { pos.shift(), newn }; - } catch (...) { - if (!mutate) node_t::delete_inner_r_e(newn); + return std::make_tuple(pos.shift(), newn); + } + IMMER_CATCH (...) { + if (!mutate) + node_t::delete_inner_r_e(newn); else { // restore the regular node that we were // attempting to relax... @@ -1286,7 +1366,7 @@ struct slice_left_mut_visitor node->impl.d.data.inner.relaxed); node->impl.d.data.inner.relaxed = nullptr; } - throw; + IMMER_RETHROW; } } } @@ -1297,31 +1377,32 @@ struct slice_left_mut_visitor auto node = pos.node(); auto idx = pos.index(first); auto count = pos.count(); - auto mutate = Mutating - && std::is_nothrow_move_constructible::value - && node->can_mutate(e); + auto mutate = Mutating && + std::is_nothrow_move_constructible::value && + node->can_mutate(e); if (mutate) { - auto data = node->leaf(); + auto data = node->leaf(); auto newcount = count - idx; std::move(data + idx, data + count, data); - destroy_n(data + newcount, idx); - return { 0, node }; + detail::destroy_n(data + newcount, idx); + return std::make_tuple(0, node); } else { auto newn = node_t::copy_leaf_e(e, node, idx, count); - if (Mutating) pos.visit(dec_visitor{}); - return { 0, newn }; + if (Mutating) + pos.visit(dec_visitor{}); + return std::make_tuple(0, newn); } } }; -template +template struct slice_left_visitor : visitor_base> { using node_t = NodeT; using this_t = slice_left_visitor; // returns a new shift and new root - using result_t = std::tuple; + using result_t = std::tuple; using no_collapse_t = slice_left_visitor; static constexpr auto B = NodeT::bits; @@ -1330,36 +1411,41 @@ struct slice_left_visitor : visitor_base> template static result_t visit_inner(PosT&& pos, size_t first) { - auto idx = pos.subindex(first); - auto count = pos.count(); - auto left_size = pos.size_before(idx); - auto child_size = pos.size_sbh(idx, left_size); - auto dropped_size = first; + auto idx = pos.subindex(first); + auto count = pos.count(); + auto left_size = pos.size_before(idx); + auto child_size = pos.size_sbh(idx, left_size); + auto dropped_size = first; auto child_dropped_size = dropped_size - left_size; if (Collapse && pos.shift() > BL && idx == pos.count() - 1) { return pos.towards_sub_oh(this_t{}, first, idx); } else { using std::get; - auto n = pos.node(); - auto newc = count - idx; - auto newn = node_t::make_inner_r_n(newc); - try { - auto subs = pos.towards_sub_oh(no_collapse_t{}, first, idx); - auto newr = newn->relaxed(); + auto n = pos.node(); + auto newc = count - idx; + auto newn = node_t::make_inner_r_n(newc); + IMMER_TRY { + auto subs = pos.towards_sub_oh(no_collapse_t{}, first, idx); + auto newr = newn->relaxed(); newr->d.count = count - idx; newr->d.sizes[0] = child_size - child_dropped_size; - pos.copy_sizes(idx + 1, newr->d.count - 1, - newr->d.sizes[0], newr->d.sizes + 1); - assert(newr->d.sizes[newr->d.count - 1] == pos.size() - dropped_size); + assert(newr->d.sizes[0]); + pos.copy_sizes(idx + 1, + newr->d.count - 1, + newr->d.sizes[0], + newr->d.sizes + 1); + assert(newr->d.sizes[newr->d.count - 1] == + pos.size() - dropped_size); newn->inner()[0] = get<1>(subs); - std::uninitialized_copy(n->inner() + idx + 1, - n->inner() + count, - newn->inner() + 1); + std::copy(n->inner() + idx + 1, + n->inner() + count, + newn->inner() + 1); node_t::inc_nodes(newn->inner() + 1, newr->d.count - 1); - return { pos.shift(), newn }; - } catch (...) { + return std::make_tuple(pos.shift(), newn); + } + IMMER_CATCH (...) { node_t::delete_inner_r(newn, newc); - throw; + IMMER_RETHROW; } } } @@ -1368,7 +1454,7 @@ struct slice_left_visitor : visitor_base> static result_t visit_leaf(PosT&& pos, size_t first) { auto n = node_t::copy_leaf(pos.node(), pos.index(first), pos.count()); - return { 0, n }; + return std::make_tuple(0, n); } }; @@ -1386,54 +1472,71 @@ struct concat_center_pos shift_t shift_ = 0u; count_t count_ = 0u; node_t* nodes_[max_children]; - size_t sizes_[max_children]; + size_t sizes_[max_children]; auto shift() const { return shift_; } - concat_center_pos(shift_t s, - Node* n0, size_t s0) - : shift_{s}, count_{1}, nodes_{n0}, sizes_{s0} {} + concat_center_pos(shift_t s, Node* n0, size_t s0) + : shift_{s} + , count_{1} + , nodes_{n0} + , sizes_{s0} + {} - concat_center_pos(shift_t s, - Node* n0, size_t s0, - Node* n1, size_t s1) - : shift_{s}, count_{2}, nodes_{n0, n1}, sizes_{s0, s1} {} + concat_center_pos(shift_t s, Node* n0, size_t s0, Node* n1, size_t s1) + : shift_{s} + , count_{2} + , nodes_{n0, n1} + , sizes_{s0, s0 + s1} + {} concat_center_pos(shift_t s, - Node* n0, size_t s0, - Node* n1, size_t s1, - Node* n2, size_t s2) - : shift_{s}, count_{3}, nodes_{n0, n1, n2}, sizes_{s0, s1, s2} {} + Node* n0, + size_t s0, + Node* n1, + size_t s1, + Node* n2, + size_t s2) + : shift_{s} + , count_{3} + , nodes_{n0, n1, n2} + , sizes_{s0, s0 + s1, s0 + s1 + s2} + {} template - void each_sub(Visitor v, Args&& ...args) + void each_sub(Visitor v, Args&&... args) { if (shift_ == BL) { - for (auto i = count_t{0}; i < count_; ++i) - make_leaf_sub_pos(nodes_[i], sizes_[i]).visit(v, args...); + auto s = size_t{}; + for (auto i = count_t{0}; i < count_; ++i) { + make_leaf_sub_pos(nodes_[i], sizes_[i] - s).visit(v, args...); + s = sizes_[i]; + } } else { for (auto i = count_t{0}; i < count_; ++i) - make_relaxed_pos(nodes_[i], shift_ - B, nodes_[i]->relaxed()).visit(v, args...); + make_relaxed_pos(nodes_[i], shift_ - B, nodes_[i]->relaxed()) + .visit(v, args...); } } relaxed_pos realize() && { if (count_ > 1) { - try { + IMMER_TRY { auto result = node_t::make_inner_r_n(count_); auto r = result->relaxed(); - r->d.count = count_; + r->d.count = count_; std::copy(nodes_, nodes_ + count_, result->inner()); std::copy(sizes_, sizes_ + count_, r->d.sizes); - return { result, shift_, r }; - } catch (...) { + return {result, shift_, r}; + } + IMMER_CATCH (...) { each_sub(dec_visitor{}); - throw; + IMMER_RETHROW; } } else { assert(shift_ >= B + BL); - return { nodes_[0], shift_ - B, nodes_[0]->relaxed() }; + return {nodes_[0], shift_ - B, nodes_[0]->relaxed()}; } } @@ -1442,13 +1545,13 @@ struct concat_center_pos if (count_ > 1) { auto result = node_t::make_inner_r_e(e); auto r = result->relaxed(); - r->d.count = count_; + r->d.count = count_; std::copy(nodes_, nodes_ + count_, result->inner()); std::copy(sizes_, sizes_ + count_, r->d.sizes); - return { result, shift_, r }; + return {result, shift_, r}; } else { assert(shift_ >= B + BL); - return { nodes_[0], shift_ - B, nodes_[0]->relaxed() }; + return {nodes_[0], shift_ - B, nodes_[0]->relaxed()}; } } }; @@ -1456,28 +1559,30 @@ struct concat_center_pos template struct concat_merger { - using node_t = Node; + using node_t = Node; static constexpr auto B = Node::bits; static constexpr auto BL = Node::bits_leaf; using result_t = concat_center_pos; count_t* curr_; - count_t n_; + count_t n_; result_t result_; concat_merger(shift_t shift, count_t* counts, count_t n) : curr_{counts} , n_{n} - , result_{shift + B, node_t::make_inner_r_n(std::min(n_, branches)), 0} + , result_{ + shift + B, node_t::make_inner_r_n(std::min(n_, branches)), 0} {} - node_t* to_ = {}; - count_t to_offset_ = {}; - size_t to_size_ = {}; + node_t* to_ = {}; + count_t to_offset_ = {}; + size_t to_size_ = {}; void add_child(node_t* p, size_t size) { + assert(size); ++curr_; auto parent = result_.nodes_[result_.count_ - 1]; auto relaxed = parent->relaxed(); @@ -1488,12 +1593,15 @@ struct concat_merger relaxed = parent->relaxed(); result_.nodes_[result_.count_] = parent; result_.sizes_[result_.count_] = result_.sizes_[result_.count_ - 1]; + assert(result_.sizes_[result_.count_]); ++result_.count_; } auto idx = relaxed->d.count++; result_.sizes_[result_.count_ - 1] += size; + assert(result_.sizes_[result_.count_ - 1]); relaxed->d.sizes[idx] = size + (idx ? relaxed->d.sizes[idx - 1] : 0); - parent->inner() [idx] = p; + assert(relaxed->d.sizes[idx]); + parent->inner()[idx] = p; }; template @@ -1511,16 +1619,16 @@ struct concat_merger auto from_data = from->leaf(); do { if (!to_) { - to_ = node_t::make_leaf_n(*curr_); + to_ = node_t::make_leaf_n(*curr_); to_offset_ = 0; } auto data = to_->leaf(); - auto to_copy = std::min(from_count - from_offset, - *curr_ - to_offset_); - std::uninitialized_copy(from_data + from_offset, - from_data + from_offset + to_copy, - data + to_offset_); - to_offset_ += to_copy; + auto to_copy = + std::min(from_count - from_offset, *curr_ - to_offset_); + detail::uninitialized_copy(from_data + from_offset, + from_data + from_offset + to_copy, + data + to_offset_); + to_offset_ += to_copy; from_offset += to_copy; if (*curr_ == to_offset_) { add_child(to_, to_offset_); @@ -1542,26 +1650,27 @@ struct concat_merger from->inc(); } else { auto from_offset = count_t{}; - auto from_data = from->inner(); + auto from_data = from->inner(); do { if (!to_) { - to_ = node_t::make_inner_r_n(*curr_); + to_ = node_t::make_inner_r_n(*curr_); to_offset_ = 0; to_size_ = 0; } - auto data = to_->inner(); - auto to_copy = std::min(from_count - from_offset, - *curr_ - to_offset_); - std::uninitialized_copy(from_data + from_offset, - from_data + from_offset + to_copy, - data + to_offset_); + auto data = to_->inner(); + auto to_copy = + std::min(from_count - from_offset, *curr_ - to_offset_); + std::copy(from_data + from_offset, + from_data + from_offset + to_copy, + data + to_offset_); node_t::inc_nodes(from_data + from_offset, to_copy); - auto sizes = to_->relaxed()->d.sizes; - p.copy_sizes(from_offset, to_copy, - to_size_, sizes + to_offset_); - to_offset_ += to_copy; + auto sizes = to_->relaxed()->d.sizes; + p.copy_sizes( + from_offset, to_copy, to_size_, sizes + to_offset_); + to_offset_ += to_copy; from_offset += to_copy; - to_size_ = sizes[to_offset_ - 1]; + to_size_ = sizes[to_offset_ - 1]; + assert(to_size_); if (*curr_ == to_offset_) { to_->relaxed()->d.count = to_offset_; add_child(to_, to_size_); @@ -1598,11 +1707,15 @@ struct concat_merger_visitor : visitor_base template static void visit_inner(Pos&& p, Merger& merger) - { merger.merge_inner(p); } + { + merger.merge_inner(p); + } template static void visit_leaf(Pos&& p, Merger& merger) - { merger.merge_leaf(p); } + { + merger.merge_leaf(p); + } }; struct concat_rebalance_plan_fill_visitor @@ -1625,8 +1738,8 @@ struct concat_rebalance_plan { static constexpr auto max_children = 2 * branches + 1; - count_t counts [max_children]; - count_t n = 0u; + count_t counts[max_children]; + count_t n = 0u; count_t total = 0u; template @@ -1649,19 +1762,23 @@ struct concat_rebalance_plan #endif constexpr count_t rrb_extras = 2; constexpr count_t rrb_invariant = 1; - const auto bits = shift == BL ? BL : B; - const auto branches = count_t{1} << bits; - const auto optimal = ((total - 1) >> bits) + 1; - count_t i = 0; + const auto bits = shift == BL ? BL : B; + const auto branches = count_t{1} << bits; + const auto optimal = ((total - 1) >> bits) + 1; + count_t i = 0; while (n >= optimal + rrb_extras) { // skip ok nodes - while (counts[i] > branches - rrb_invariant) i++; + while (counts[i] > branches - rrb_invariant) + i++; + assert(i < n); // short node, redistribute auto remaining = counts[i]; do { - auto count = std::min(remaining + counts[i+1], branches); - counts[i] = count; - remaining += counts[i + 1] - count; + auto next = counts[i + 1]; + auto count = std::min(remaining + next, branches); + counts[i] = count; + assert(counts[i]); + remaining += next - count; ++i; } while (remaining > 0); // remove node @@ -1681,38 +1798,38 @@ struct concat_rebalance_plan using node_t = node_type; using merger_t = concat_merger; using visitor_t = concat_merger_visitor; - auto merger = merger_t{cpos.shift(), counts, n}; - try { + auto merger = merger_t{cpos.shift(), counts, n}; + IMMER_TRY { lpos.each_left_sub(visitor_t{}, merger); cpos.each_sub(visitor_t{}, merger); rpos.each_right_sub(visitor_t{}, merger); cpos.each_sub(dec_visitor{}); return merger.finish(); - } catch (...) { + } + IMMER_CATCH (...) { merger.abort(); - throw; + IMMER_RETHROW; } } }; template -concat_center_pos -concat_rebalance(LPos&& lpos, CPos&& cpos, RPos&& rpos) +concat_center_pos concat_rebalance(LPos&& lpos, CPos&& cpos, RPos&& rpos) { auto plan = concat_rebalance_plan{}; plan.fill(lpos, cpos, rpos); plan.shuffle(cpos.shift()); - try { + IMMER_TRY { return plan.merge(lpos, cpos, rpos); - } catch (...) { + } + IMMER_CATCH (...) { cpos.each_sub(dec_visitor{}); - throw; + IMMER_RETHROW; } } template -concat_center_pos -concat_leafs(LPos&& lpos, TPos&& tpos, RPos&& rpos) +concat_center_pos concat_leafs(LPos&& lpos, TPos&& tpos, RPos&& rpos) { static_assert(Node::bits >= 2, ""); assert(lpos.shift() == tpos.shift()); @@ -1721,15 +1838,20 @@ concat_leafs(LPos&& lpos, TPos&& tpos, RPos&& rpos) if (tpos.count() > 0) return { Node::bits_leaf, - lpos.node()->inc(), lpos.count(), - tpos.node()->inc(), tpos.count(), - rpos.node()->inc(), rpos.count(), + lpos.node()->inc(), + lpos.count(), + tpos.node()->inc(), + tpos.count(), + rpos.node()->inc(), + rpos.count(), }; else return { Node::bits_leaf, - lpos.node()->inc(), lpos.count(), - rpos.node()->inc(), rpos.count(), + lpos.node()->inc(), + lpos.count(), + rpos.node()->inc(), + rpos.count(), }; } @@ -1741,8 +1863,7 @@ template struct concat_both_visitor; template -concat_center_pos -concat_inners(LPos&& lpos, TPos&& tpos, RPos&& rpos) +concat_center_pos concat_inners(LPos&& lpos, TPos&& tpos, RPos&& rpos) { auto lshift = lpos.shift(); auto rshift = rpos.shift(); @@ -1768,12 +1889,16 @@ struct concat_left_visitor : visitor_base> template static concat_center_pos visit_inner(LPos&& lpos, TPos&& tpos, RPos&& rpos) - { return concat_inners(lpos, tpos, rpos); } + { + return concat_inners(lpos, tpos, rpos); + } template static concat_center_pos visit_leaf(LPos&& lpos, TPos&& tpos, RPos&& rpos) - { IMMER_UNREACHABLE; } + { + IMMER_UNREACHABLE; + } }; template @@ -1784,29 +1909,36 @@ struct concat_right_visitor : visitor_base> template static concat_center_pos visit_inner(RPos&& rpos, LPos&& lpos, TPos&& tpos) - { return concat_inners(lpos, tpos, rpos); } + { + return concat_inners(lpos, tpos, rpos); + } template static concat_center_pos visit_leaf(RPos&& rpos, LPos&& lpos, TPos&& tpos) - { return concat_leafs(lpos, tpos, rpos); } + { + return concat_leafs(lpos, tpos, rpos); + } }; template -struct concat_both_visitor - : visitor_base> +struct concat_both_visitor : visitor_base> { using this_t = concat_both_visitor; template static concat_center_pos visit_inner(LPos&& lpos, TPos&& tpos, RPos&& rpos) - { return rpos.first_sub(concat_right_visitor{}, lpos, tpos); } + { + return rpos.first_sub(concat_right_visitor{}, lpos, tpos); + } template static concat_center_pos visit_leaf(LPos&& lpos, TPos&& tpos, RPos&& rpos) - { return rpos.first_sub_leaf(concat_right_visitor{}, lpos, tpos); } + { + return rpos.first_sub_leaf(concat_right_visitor{}, lpos, tpos); + } }; template @@ -1818,47 +1950,56 @@ struct concat_trees_right_visitor template static concat_center_pos visit_node(RPos&& rpos, LPos&& lpos, TPos&& tpos) - { return concat_inners(lpos, tpos, rpos); } + { + return concat_inners(lpos, tpos, rpos); + } }; template -struct concat_trees_left_visitor - : visitor_base> +struct concat_trees_left_visitor : visitor_base> { using this_t = concat_trees_left_visitor; template static concat_center_pos - visit_node(LPos&& lpos, TPos&& tpos, Args&& ...args) - { return visit_maybe_relaxed_sub( - args..., - concat_trees_right_visitor{}, - lpos, tpos); } + visit_node(LPos&& lpos, TPos&& tpos, Args&&... args) + { + return visit_maybe_relaxed_sub( + args..., concat_trees_right_visitor{}, lpos, tpos); + } }; template -relaxed_pos -concat_trees(Node* lroot, shift_t lshift, size_t lsize, - Node* ltail, count_t ltcount, - Node* rroot, shift_t rshift, size_t rsize) +relaxed_pos concat_trees(Node* lroot, + shift_t lshift, + size_t lsize, + Node* ltail, + count_t ltcount, + Node* rroot, + shift_t rshift, + size_t rsize) { - return visit_maybe_relaxed_sub( - lroot, lshift, lsize, - concat_trees_left_visitor{}, - make_leaf_pos(ltail, ltcount), - rroot, rshift, rsize) + return visit_maybe_relaxed_sub(lroot, + lshift, + lsize, + concat_trees_left_visitor{}, + make_leaf_pos(ltail, ltcount), + rroot, + rshift, + rsize) .realize(); } template -relaxed_pos -concat_trees(Node* ltail, count_t ltcount, - Node* rroot, shift_t rshift, size_t rsize) +relaxed_pos concat_trees( + Node* ltail, count_t ltcount, Node* rroot, shift_t rshift, size_t rsize) { - return make_singleton_regular_sub_pos(ltail, ltcount).visit( - concat_trees_left_visitor{}, - empty_leaf_pos{}, - rroot, rshift, rsize) + return make_singleton_regular_sub_pos(ltail, ltcount) + .visit(concat_trees_left_visitor{}, + empty_leaf_pos{}, + rroot, + rshift, + rsize) .realize(); } @@ -1876,18 +2017,21 @@ struct concat_merger_mut using result_t = concat_center_pos; - edit_t ec_ = {}; + edit_t ec_ = {}; count_t* curr_; - count_t n_; + count_t n_; result_t result_; - count_t count_ = 0; - node_t* candidate_ = nullptr; - edit_t candidate_e_ = Node::memory::transience_t::noone; - - concat_merger_mut(edit_t ec, shift_t shift, - count_t* counts, count_t n, - edit_t candidate_e, node_t* candidate) + count_t count_ = 0; + node_t* candidate_ = nullptr; + edit_t candidate_e_ = Node::memory::transience_t::noone; + + concat_merger_mut(edit_t ec, + shift_t shift, + count_t* counts, + count_t n, + edit_t candidate_e, + node_t* candidate) : ec_{ec} , curr_{counts} , n_{n} @@ -1901,15 +2045,19 @@ struct concat_merger_mut } } - node_t* to_ = {}; - count_t to_offset_ = {}; - size_t to_size_ = {}; + node_t* to_ = {}; + count_t to_offset_ = {}; + size_t to_size_ = {}; void set_candidate(edit_t candidate_e, node_t* candidate) - { candidate_ = candidate; candidate_e_ = candidate_e; } + { + candidate_ = candidate; + candidate_e_ = candidate_e; + } void add_child(node_t* p, size_t size) { + assert(size); ++curr_; auto parent = result_.nodes_[result_.count_ - 1]; auto relaxed = parent->relaxed(); @@ -1922,17 +2070,21 @@ struct concat_merger_mut parent->ensure_mutable_relaxed_e(candidate_e_, ec_); candidate_ = nullptr; } else - parent = node_t::make_inner_r_e(ec_); - count_ = 0; - relaxed = parent->relaxed(); + parent = node_t::make_inner_r_e(ec_); + count_ = 0; + relaxed = parent->relaxed(); result_.nodes_[result_.count_] = parent; result_.sizes_[result_.count_] = result_.sizes_[result_.count_ - 1]; + assert(result_.sizes_[result_.count_]); ++result_.count_; } auto idx = count_++; result_.sizes_[result_.count_ - 1] += size; + assert(size); + assert(result_.sizes_[result_.count_ - 1]); relaxed->d.sizes[idx] = size + (idx ? relaxed->d.sizes[idx - 1] : 0); - parent->inner() [idx] = p; + assert(relaxed->d.sizes[idx]); + parent->inner()[idx] = p; }; template @@ -1941,18 +2093,19 @@ struct concat_merger_mut auto from = p.node(); auto from_size = p.size(); auto from_count = p.count(); + assert(from); assert(from_size); if (!to_ && *curr_ == from_count) { add_child(from, from_size); } else { - auto from_offset = count_t{}; - auto from_data = from->leaf(); - auto from_mutate = from->can_mutate(e); + auto from_offset = count_t{}; + auto from_data = from->leaf(); + auto from_mutate = from->can_mutate(e); do { if (!to_) { if (from_mutate) { node_t::ownee(from) = ec_; - to_ = from->inc(); + to_ = from->inc(); assert(from_count); } else { to_ = node_t::make_leaf_e(ec_); @@ -1960,8 +2113,8 @@ struct concat_merger_mut to_offset_ = 0; } auto data = to_->leaf(); - auto to_copy = std::min(from_count - from_offset, - *curr_ - to_offset_); + auto to_copy = + std::min(from_count - from_offset, *curr_ - to_offset_); if (from == to_) { if (from_offset != to_offset_) std::move(from_data + from_offset, @@ -1969,15 +2122,17 @@ struct concat_merger_mut data + to_offset_); } else { if (!from_mutate) - std::uninitialized_copy(from_data + from_offset, - from_data + from_offset + to_copy, - data + to_offset_); + detail::uninitialized_copy(from_data + from_offset, + from_data + from_offset + + to_copy, + data + to_offset_); else detail::uninitialized_move(from_data + from_offset, - from_data + from_offset + to_copy, - data + to_offset_); + from_data + from_offset + + to_copy, + data + to_offset_); } - to_offset_ += to_copy; + to_offset_ += to_copy; from_offset += to_copy; if (*curr_ == to_offset_) { add_child(to_, to_offset_); @@ -1997,8 +2152,8 @@ struct concat_merger_mut if (!to_ && *curr_ == from_count) { add_child(from, from_size); } else { - auto from_offset = count_t{}; - auto from_data = from->inner(); + auto from_offset = count_t{}; + auto from_data = from->inner(); auto from_mutate = from->can_relax() && from->can_mutate(e); do { if (!to_) { @@ -2012,20 +2167,20 @@ struct concat_merger_mut to_offset_ = 0; to_size_ = 0; } - auto data = to_->inner(); - auto to_copy = std::min(from_count - from_offset, - *curr_ - to_offset_); - auto sizes = to_->relaxed()->d.sizes; + auto data = to_->inner(); + auto to_copy = + std::min(from_count - from_offset, *curr_ - to_offset_); + auto sizes = to_->relaxed()->d.sizes; if (from != to_ || from_offset != to_offset_) { std::copy(from_data + from_offset, from_data + from_offset + to_copy, data + to_offset_); - p.copy_sizes(from_offset, to_copy, - to_size_, sizes + to_offset_); + p.copy_sizes( + from_offset, to_copy, to_size_, sizes + to_offset_); } - to_offset_ += to_copy; + to_offset_ += to_copy; from_offset += to_copy; - to_size_ = sizes[to_offset_ - 1]; + to_size_ = sizes[to_offset_ - 1]; if (*curr_ == to_offset_) { to_->relaxed()->d.count = to_offset_; add_child(to_, to_size_); @@ -2057,14 +2212,16 @@ struct concat_merger_mut_visitor : visitor_base using this_t = concat_merger_mut_visitor; template - static void visit_inner(Pos&& p, - Merger& merger, edit_type e) - { merger.merge_inner(p, e); } + static void visit_inner(Pos&& p, Merger& merger, edit_type e) + { + merger.merge_inner(p, e); + } template - static void visit_leaf(Pos&& p, - Merger& merger, edit_type e) - { merger.merge_leaf(p, e); } + static void visit_leaf(Pos&& p, Merger& merger, edit_type e) + { + merger.merge_leaf(p, e); + } }; template @@ -2073,40 +2230,48 @@ struct concat_rebalance_plan_mut : concat_rebalance_plan using this_t = concat_rebalance_plan_mut; template - concat_center_mut_pos> - merge(edit_type ec, - edit_type el, LPos&& lpos, CPos&& cpos, - edit_type er, RPos&& rpos) + concat_center_mut_pos> merge(edit_type ec, + edit_type el, + LPos&& lpos, + CPos&& cpos, + edit_type er, + RPos&& rpos) { using node_t = node_type; using merger_t = concat_merger_mut; using visitor_t = concat_merger_mut_visitor; - auto lnode = ((node_t*)lpos.node()); - auto rnode = ((node_t*)rpos.node()); - auto lmut2 = lnode && lnode->can_relax() && lnode->can_mutate(el); - auto rmut2 = rnode && rnode->can_relax() && rnode->can_mutate(er); - auto merger = merger_t{ - ec, cpos.shift(), this->counts, this->n, - el, lmut2 ? lnode : nullptr - }; - try { + auto lnode = ((node_t*) lpos.node()); + auto rnode = ((node_t*) rpos.node()); + auto lmut2 = lnode && lnode->can_relax() && lnode->can_mutate(el); + auto rmut2 = rnode && rnode->can_relax() && rnode->can_mutate(er); + auto merger = merger_t{ec, + cpos.shift(), + this->counts, + this->n, + el, + lmut2 ? lnode : nullptr}; + IMMER_TRY { lpos.each_left_sub(visitor_t{}, merger, el); cpos.each_sub(visitor_t{}, merger, ec); - if (rmut2) merger.set_candidate(er, rnode); + if (rmut2) + merger.set_candidate(er, rnode); rpos.each_right_sub(visitor_t{}, merger, er); return merger.finish(); - } catch (...) { + } + IMMER_CATCH (...) { merger.abort(); - throw; + IMMER_RETHROW; } } }; template -concat_center_pos -concat_rebalance_mut(edit_type ec, - edit_type el, LPos&& lpos, CPos&& cpos, - edit_type er, RPos&& rpos) +concat_center_pos concat_rebalance_mut(edit_type ec, + edit_type el, + LPos&& lpos, + CPos&& cpos, + edit_type er, + RPos&& rpos) { auto plan = concat_rebalance_plan_mut{}; plan.fill(lpos, cpos, rpos); @@ -2115,10 +2280,12 @@ concat_rebalance_mut(edit_type ec, } template -concat_center_mut_pos -concat_leafs_mut(edit_type ec, - edit_type el, LPos&& lpos, TPos&& tpos, - edit_type er, RPos&& rpos) +concat_center_mut_pos concat_leafs_mut(edit_type ec, + edit_type el, + LPos&& lpos, + TPos&& tpos, + edit_type er, + RPos&& rpos) { static_assert(Node::bits >= 2, ""); assert(lpos.shift() == tpos.shift()); @@ -2127,15 +2294,20 @@ concat_leafs_mut(edit_type ec, if (tpos.count() > 0) return { Node::bits_leaf, - lpos.node(), lpos.count(), - tpos.node(), tpos.count(), - rpos.node(), rpos.count(), + lpos.node(), + lpos.count(), + tpos.node(), + tpos.count(), + rpos.node(), + rpos.count(), }; else return { Node::bits_leaf, - lpos.node(), lpos.count(), - rpos.node(), rpos.count(), + lpos.node(), + lpos.count(), + rpos.node(), + rpos.count(), }; } @@ -2147,35 +2319,33 @@ template struct concat_both_mut_visitor; template -concat_center_mut_pos -concat_inners_mut(edit_type ec, - edit_type el, LPos&& lpos, TPos&& tpos, - edit_type er, RPos&& rpos) +concat_center_mut_pos concat_inners_mut(edit_type ec, + edit_type el, + LPos&& lpos, + TPos&& tpos, + edit_type er, + RPos&& rpos) { auto lshift = lpos.shift(); auto rshift = rpos.shift(); // lpos.node() can be null it is a singleton_regular_sub_pos<...>, // this is, when the tree is just a tail... if (lshift > rshift) { - auto cpos = lpos.last_sub(concat_left_mut_visitor{}, - ec, el, tpos, er, rpos); - return concat_rebalance_mut(ec, - el, lpos, cpos, - er, null_sub_pos{}); + auto cpos = lpos.last_sub( + concat_left_mut_visitor{}, ec, el, tpos, er, rpos); + return concat_rebalance_mut( + ec, el, lpos, cpos, er, null_sub_pos{}); } else if (lshift < rshift) { - auto cpos = rpos.first_sub(concat_right_mut_visitor{}, - ec, el, lpos, tpos, er); - return concat_rebalance_mut(ec, - el, null_sub_pos{}, cpos, - er, rpos); + auto cpos = rpos.first_sub( + concat_right_mut_visitor{}, ec, el, lpos, tpos, er); + return concat_rebalance_mut( + ec, el, null_sub_pos{}, cpos, er, rpos); } else { assert(lshift == rshift); assert(Node::bits_leaf == 0u || lshift > 0); - auto cpos = lpos.last_sub(concat_both_mut_visitor{}, - ec, el, tpos, er, rpos); - return concat_rebalance_mut(ec, - el, lpos, cpos, - er, rpos); + auto cpos = lpos.last_sub( + concat_both_mut_visitor{}, ec, el, tpos, er, rpos); + return concat_rebalance_mut(ec, el, lpos, cpos, er, rpos); } } @@ -2186,67 +2356,62 @@ struct concat_left_mut_visitor : visitor_base> using edit_t = typename Node::edit_t; template - static concat_center_mut_pos - visit_inner(LPos&& lpos, edit_t ec, - edit_t el, TPos&& tpos, - edit_t er, RPos&& rpos) - { return concat_inners_mut( - ec, el, lpos, tpos, er, rpos); } + static concat_center_mut_pos visit_inner( + LPos&& lpos, edit_t ec, edit_t el, TPos&& tpos, edit_t er, RPos&& rpos) + { + return concat_inners_mut(ec, el, lpos, tpos, er, rpos); + } template - static concat_center_mut_pos - visit_leaf(LPos&& lpos, edit_t ec, - edit_t el, TPos&& tpos, - edit_t er, RPos&& rpos) - { IMMER_UNREACHABLE; } + static concat_center_mut_pos visit_leaf( + LPos&& lpos, edit_t ec, edit_t el, TPos&& tpos, edit_t er, RPos&& rpos) + { + IMMER_UNREACHABLE; + } }; template -struct concat_right_mut_visitor - : visitor_base> +struct concat_right_mut_visitor : visitor_base> { using this_t = concat_right_mut_visitor; using edit_t = typename Node::edit_t; template - static concat_center_mut_pos - visit_inner(RPos&& rpos, edit_t ec, - edit_t el, LPos&& lpos, TPos&& tpos, - edit_t er) - { return concat_inners_mut( - ec, el, lpos, tpos, er, rpos); } + static concat_center_mut_pos visit_inner( + RPos&& rpos, edit_t ec, edit_t el, LPos&& lpos, TPos&& tpos, edit_t er) + { + return concat_inners_mut(ec, el, lpos, tpos, er, rpos); + } template - static concat_center_mut_pos - visit_leaf(RPos&& rpos, edit_t ec, - edit_t el, LPos&& lpos, TPos&& tpos, - edit_t er) - { return concat_leafs_mut( - ec, el, lpos, tpos, er, rpos); } + static concat_center_mut_pos visit_leaf( + RPos&& rpos, edit_t ec, edit_t el, LPos&& lpos, TPos&& tpos, edit_t er) + { + return concat_leafs_mut(ec, el, lpos, tpos, er, rpos); + } }; template -struct concat_both_mut_visitor - : visitor_base> +struct concat_both_mut_visitor : visitor_base> { using this_t = concat_both_mut_visitor; using edit_t = typename Node::edit_t; template - static concat_center_mut_pos - visit_inner(LPos&& lpos, edit_t ec, - edit_t el, TPos&& tpos, - edit_t er, RPos&& rpos) - { return rpos.first_sub(concat_right_mut_visitor{}, - ec, el, lpos, tpos, er); } + static concat_center_mut_pos visit_inner( + LPos&& lpos, edit_t ec, edit_t el, TPos&& tpos, edit_t er, RPos&& rpos) + { + return rpos.first_sub( + concat_right_mut_visitor{}, ec, el, lpos, tpos, er); + } template - static concat_center_mut_pos - visit_leaf(LPos&& lpos, edit_t ec, - edit_t el, TPos&& tpos, - edit_t er, RPos&& rpos) - { return rpos.first_sub_leaf(concat_right_mut_visitor{}, - ec, el, lpos, tpos, er); } + static concat_center_mut_pos visit_leaf( + LPos&& lpos, edit_t ec, edit_t el, TPos&& tpos, edit_t er, RPos&& rpos) + { + return rpos.first_sub_leaf( + concat_right_mut_visitor{}, ec, el, lpos, tpos, er); + } }; template @@ -2257,12 +2422,11 @@ struct concat_trees_right_mut_visitor using edit_t = typename Node::edit_t; template - static concat_center_mut_pos - visit_node(RPos&& rpos, edit_t ec, - edit_t el, LPos&& lpos, TPos&& tpos, - edit_t er) - { return concat_inners_mut( - ec, el, lpos, tpos, er, rpos); } + static concat_center_mut_pos visit_node( + RPos&& rpos, edit_t ec, edit_t el, LPos&& lpos, TPos&& tpos, edit_t er) + { + return concat_inners_mut(ec, el, lpos, tpos, er, rpos); + } }; template @@ -2273,47 +2437,69 @@ struct concat_trees_left_mut_visitor using edit_t = typename Node::edit_t; template - static concat_center_mut_pos - visit_node(LPos&& lpos, edit_t ec, - edit_t el, TPos&& tpos, - edit_t er, Args&& ...args) - { return visit_maybe_relaxed_sub( - args..., - concat_trees_right_mut_visitor{}, - ec, el, lpos, tpos, er); } + static concat_center_mut_pos visit_node(LPos&& lpos, + edit_t ec, + edit_t el, + TPos&& tpos, + edit_t er, + Args&&... args) + { + return visit_maybe_relaxed_sub(args..., + concat_trees_right_mut_visitor{}, + ec, + el, + lpos, + tpos, + er); + } }; template -relaxed_pos -concat_trees_mut(edit_type ec, - edit_type el, - Node* lroot, shift_t lshift, size_t lsize, - Node* ltail, count_t ltcount, - edit_type er, - Node* rroot, shift_t rshift, size_t rsize) +relaxed_pos concat_trees_mut(edit_type ec, + edit_type el, + Node* lroot, + shift_t lshift, + size_t lsize, + Node* ltail, + count_t ltcount, + edit_type er, + Node* rroot, + shift_t rshift, + size_t rsize) { - return visit_maybe_relaxed_sub( - lroot, lshift, lsize, - concat_trees_left_mut_visitor{}, - ec, - el, make_leaf_pos(ltail, ltcount), - er, rroot, rshift, rsize) + return visit_maybe_relaxed_sub(lroot, + lshift, + lsize, + concat_trees_left_mut_visitor{}, + ec, + el, + make_leaf_pos(ltail, ltcount), + er, + rroot, + rshift, + rsize) .realize_e(ec); } template -relaxed_pos -concat_trees_mut(edit_type ec, - edit_type el, - Node* ltail, count_t ltcount, - edit_type er, - Node* rroot, shift_t rshift, size_t rsize) +relaxed_pos concat_trees_mut(edit_type ec, + edit_type el, + Node* ltail, + count_t ltcount, + edit_type er, + Node* rroot, + shift_t rshift, + size_t rsize) { - return make_singleton_regular_sub_pos(ltail, ltcount).visit( - concat_trees_left_mut_visitor{}, - ec, - el, empty_leaf_pos{}, - er, rroot, rshift, rsize) + return make_singleton_regular_sub_pos(ltail, ltcount) + .visit(concat_trees_left_mut_visitor{}, + ec, + el, + empty_leaf_pos{}, + er, + rroot, + rshift, + rsize) .realize_e(ec); } diff --git a/src/immer/detail/rbts/position.hpp b/src/immer/detail/rbts/position.hpp index 4ff579f8f9..cbdd1c2f1f 100644 --- a/src/immer/detail/rbts/position.hpp +++ b/src/immer/detail/rbts/position.hpp @@ -12,8 +12,8 @@ #include #include -#include #include +#include namespace immer { namespace detail { @@ -38,17 +38,21 @@ struct empty_regular_pos node_t* node_; count_t count() const { return 0; } - node_t* node() const { return node_; } + node_t* node() const { return node_; } shift_t shift() const { return 0; } - size_t size() const { return 0; } + size_t size() const { return 0; } template - void each(Visitor, Args&&...) {} + void each(Visitor, Args&&...) + {} template - bool each_pred(Visitor, Args&&...) { return true; } + bool each_pred(Visitor, Args&&...) + { + return true; + } template - decltype(auto) visit(Visitor v, Args&& ...args) + decltype(auto) visit(Visitor v, Args&&... args) { return Visitor::visit_regular(*this, std::forward(args)...); } @@ -67,12 +71,12 @@ struct empty_leaf_pos node_t* node_; count_t count() const { return 0; } - node_t* node() const { return node_; } + node_t* node() const { return node_; } shift_t shift() const { return 0; } - size_t size() const { return 0; } + size_t size() const { return 0; } - template - decltype(auto) visit(Visitor v, Args&& ...args) + template + decltype(auto) visit(Visitor v, Args&&... args) { return Visitor::visit_leaf(*this, std::forward(args)...); } @@ -96,14 +100,14 @@ struct leaf_pos size_t size_; count_t count() const { return index(size_ - 1) + 1; } - node_t* node() const { return node_; } - size_t size() const { return size_; } + node_t* node() const { return node_; } + size_t size() const { return size_; } shift_t shift() const { return 0; } count_t index(size_t idx) const { return idx & mask; } count_t subindex(size_t idx) const { return idx; } - template - decltype(auto) visit(Visitor v, Args&& ...args) + template + decltype(auto) visit(Visitor v, Args&&... args) { return Visitor::visit_leaf(*this, std::forward(args)...); } @@ -128,14 +132,14 @@ struct leaf_sub_pos count_t count_; count_t count() const { return count_; } - node_t* node() const { return node_; } - size_t size() const { return count_; } + node_t* node() const { return node_; } + size_t size() const { return count_; } shift_t shift() const { return 0; } count_t index(size_t idx) const { return idx & mask; } count_t subindex(size_t idx) const { return idx; } - template - decltype(auto) visit(Visitor v, Args&& ...args) + template + decltype(auto) visit(Visitor v, Args&&... args) { return Visitor::visit_leaf(*this, std::forward(args)...); } @@ -158,15 +162,16 @@ struct leaf_descent_pos using node_t = NodeT; node_t* node_; - node_t* node() const { return node_; } + node_t* node() const { return node_; } shift_t shift() const { return 0; } count_t index(size_t idx) const { return idx & mask; } template - decltype(auto) descend(Args&&...) {} + decltype(auto) descend(Args&&...) + {} - template - decltype(auto) visit(Visitor v, Args&& ...args) + template + decltype(auto) visit(Visitor v, Args&&... args) { return Visitor::visit_leaf(*this, std::forward(args)...); } @@ -189,14 +194,14 @@ struct full_leaf_pos node_t* node_; count_t count() const { return branches; } - node_t* node() const { return node_; } - size_t size() const { return branches; } + node_t* node() const { return node_; } + size_t size() const { return branches; } shift_t shift() const { return 0; } count_t index(size_t idx) const { return idx & mask; } count_t subindex(size_t idx) const { return idx; } - template - decltype(auto) visit(Visitor v, Args&& ...args) + template + decltype(auto) visit(Visitor v, Args&&... args) { return Visitor::visit_leaf(*this, std::forward(args)...); } @@ -221,78 +226,111 @@ struct regular_pos size_t size_; count_t count() const { return index(size_ - 1) + 1; } - node_t* node() const { return node_; } - size_t size() const { return size_; } + node_t* node() const { return node_; } + size_t size() const { return size_; } shift_t shift() const { return shift_; } count_t index(size_t idx) const { return (idx >> shift_) & mask; } count_t subindex(size_t idx) const { return idx >> shift_; } - size_t this_size() const { return ((size_ - 1) & ~(~size_t{} << (shift_ + B))) + 1; } + size_t this_size() const + { + return ((size_ - 1) & ~(~size_t{} << (shift_ + B))) + 1; + } template void each(Visitor v, Args&&... args) - { return each_regular(*this, v, args...); } + { + return each_regular(*this, v, args...); + } template bool each_pred(Visitor v, Args&&... args) - { return each_pred_regular(*this, v, args...); } + { + return each_pred_regular(*this, v, args...); + } template bool each_pred_zip(Visitor v, node_t* other, Args&&... args) - { return each_pred_zip_regular(*this, v, other, args...); } + { + return each_pred_zip_regular(*this, v, other, args...); + } template bool each_pred_i(Visitor v, count_t i, count_t n, Args&&... args) - { return each_pred_i_regular(*this, v, i, n, args...); } + { + return each_pred_i_regular(*this, v, i, n, args...); + } template bool each_pred_right(Visitor v, count_t start, Args&&... args) - { return each_pred_right_regular(*this, v, start, args...); } + { + return each_pred_right_regular(*this, v, start, args...); + } template bool each_pred_left(Visitor v, count_t n, Args&&... args) - { return each_pred_left_regular(*this, v, n, args...); } + { + return each_pred_left_regular(*this, v, n, args...); + } template void each_i(Visitor v, count_t i, count_t n, Args&&... args) - { return each_i_regular(*this, v, i, n, args...); } + { + return each_i_regular(*this, v, i, n, args...); + } template void each_right(Visitor v, count_t start, Args&&... args) - { return each_right_regular(*this, v, start, args...); } + { + return each_right_regular(*this, v, start, args...); + } template void each_left(Visitor v, count_t n, Args&&... args) - { return each_left_regular(*this, v, n, args...); } + { + return each_left_regular(*this, v, n, args...); + } template decltype(auto) towards(Visitor v, size_t idx, Args&&... args) - { return towards_oh_ch_regular(*this, v, idx, index(idx), count(), args...); } + { + return towards_oh_ch_regular( + *this, v, idx, index(idx), count(), args...); + } template - decltype(auto) towards_oh(Visitor v, size_t idx, - count_t offset_hint, - Args&&... args) - { return towards_oh_ch_regular(*this, v, idx, offset_hint, count(), args...); } + decltype(auto) + towards_oh(Visitor v, size_t idx, count_t offset_hint, Args&&... args) + { + return towards_oh_ch_regular( + *this, v, idx, offset_hint, count(), args...); + } template - decltype(auto) towards_oh_ch(Visitor v, size_t idx, + decltype(auto) towards_oh_ch(Visitor v, + size_t idx, count_t offset_hint, count_t count_hint, Args&&... args) - { return towards_oh_ch_regular(*this, v, idx, offset_hint, count(), args...); } + { + return towards_oh_ch_regular( + *this, v, idx, offset_hint, count(), args...); + } template - decltype(auto) towards_sub_oh(Visitor v, size_t idx, - count_t offset_hint, - Args&&... args) - { return towards_sub_oh_regular(*this, v, idx, offset_hint, args...); } + decltype(auto) + towards_sub_oh(Visitor v, size_t idx, count_t offset_hint, Args&&... args) + { + return towards_sub_oh_regular(*this, v, idx, offset_hint, args...); + } template decltype(auto) last_oh(Visitor v, count_t offset_hint, Args&&... args) - { return last_oh_regular(*this, v, offset_hint, args...); } + { + return last_oh_regular(*this, v, offset_hint, args...); + } - template - decltype(auto) visit(Visitor v, Args&& ...args) + template + decltype(auto) visit(Visitor v, Args&&... args) { return Visitor::visit_regular(*this, std::forward(args)...); } @@ -303,9 +341,9 @@ void each_regular(Pos&& p, Visitor v, Args&&... args) { constexpr auto B = bits; constexpr auto BL = bits_leaf; - auto n = p.node()->inner(); - auto last = p.count() - 1; - auto e = n + last; + auto n = p.node()->inner(); + auto last = p.count() - 1; + auto e = n + last; if (p.shift() == BL) { for (; n != e; ++n) { IMMER_PREFETCH(n + 1); @@ -325,9 +363,9 @@ bool each_pred_regular(Pos&& p, Visitor v, Args&&... args) { constexpr auto B = bits; constexpr auto BL = bits_leaf; - auto n = p.node()->inner(); - auto last = p.count() - 1; - auto e = n + last; + auto n = p.node()->inner(); + auto last = p.count() - 1; + auto e = n + last; if (p.shift() == BL) { for (; n != e; ++n) { IMMER_PREFETCH(n + 1); @@ -345,15 +383,18 @@ bool each_pred_regular(Pos&& p, Visitor v, Args&&... args) } template -bool each_pred_zip_regular(Pos&& p, Visitor v, node_type* other, Args&&... args) +bool each_pred_zip_regular(Pos&& p, + Visitor v, + node_type* other, + Args&&... args) { constexpr auto B = bits; constexpr auto BL = bits_leaf; - auto n = p.node()->inner(); - auto n2 = other->inner(); + auto n = p.node()->inner(); + auto n2 = other->inner(); auto last = p.count() - 1; - auto e = n + last; + auto e = n + last; if (p.shift() == BL) { for (; n != e; ++n, ++n2) { IMMER_PREFETCH(n + 1); @@ -372,7 +413,8 @@ bool each_pred_zip_regular(Pos&& p, Visitor v, node_type* other, Args&&... } template -bool each_pred_i_regular(Pos&& p, Visitor v, count_t f, count_t l, Args&&... args) +bool each_pred_i_regular( + Pos&& p, Visitor v, count_t f, count_t l, Args&&... args) { constexpr auto B = bits; constexpr auto BL = bits_leaf; @@ -437,8 +479,8 @@ bool each_pred_left_regular(Pos&& p, Visitor v, count_t last, Args&&... args) return false; } } else { - auto n = p.node()->inner(); - auto e = n + last; + auto n = p.node()->inner(); + auto e = n + last; auto ss = p.shift() - B; for (; n != e; ++n) if (!make_full_pos(*n, ss).visit(v, args...)) @@ -454,9 +496,9 @@ bool each_pred_right_regular(Pos&& p, Visitor v, count_t start, Args&&... args) constexpr auto BL = bits_leaf; if (p.shift() == BL) { - auto n = p.node()->inner() + start; + auto n = p.node()->inner() + start; auto last = p.count() - 1; - auto e = p.node()->inner() + last; + auto e = p.node()->inner() + last; if (n <= e) { for (; n != e; ++n) { IMMER_PREFETCH(n + 1); @@ -467,10 +509,10 @@ bool each_pred_right_regular(Pos&& p, Visitor v, count_t start, Args&&... args) return false; } } else { - auto n = p.node()->inner() + start; + auto n = p.node()->inner() + start; auto last = p.count() - 1; - auto e = p.node()->inner() + last; - auto ss = p.shift() - B; + auto e = p.node()->inner() + last; + auto ss = p.shift() - B; if (n <= e) { for (; n != e; ++n) if (!make_full_pos(*n, ss).visit(v, args...)) @@ -540,8 +582,8 @@ void each_left_regular(Pos&& p, Visitor v, count_t last, Args&&... args) make_full_leaf_pos(*n).visit(v, args...); } } else { - auto n = p.node()->inner(); - auto e = n + last; + auto n = p.node()->inner(); + auto e = n + last; auto ss = p.shift() - B; for (; n != e; ++n) make_full_pos(*n, ss).visit(v, args...); @@ -555,9 +597,9 @@ void each_right_regular(Pos&& p, Visitor v, count_t start, Args&&... args) constexpr auto BL = bits_leaf; if (p.shift() == BL) { - auto n = p.node()->inner() + start; + auto n = p.node()->inner() + start; auto last = p.count() - 1; - auto e = p.node()->inner() + last; + auto e = p.node()->inner() + last; if (n <= e) { for (; n != e; ++n) { IMMER_PREFETCH(n + 1); @@ -566,10 +608,10 @@ void each_right_regular(Pos&& p, Visitor v, count_t start, Args&&... args) make_leaf_pos(*n, p.size()).visit(v, args...); } } else { - auto n = p.node()->inner() + start; + auto n = p.node()->inner() + start; auto last = p.count() - 1; - auto e = p.node()->inner() + last; - auto ss = p.shift() - B; + auto e = p.node()->inner() + last; + auto ss = p.shift() - B; if (n <= e) { for (; n != e; ++n) make_full_pos(*n, ss).visit(v, args...); @@ -579,7 +621,9 @@ void each_right_regular(Pos&& p, Visitor v, count_t start, Args&&... args) } template -decltype(auto) towards_oh_ch_regular(Pos&& p, Visitor v, size_t idx, +decltype(auto) towards_oh_ch_regular(Pos&& p, + Visitor v, + size_t idx, count_t offset_hint, count_t count_hint, Args&&... args) @@ -587,64 +631,60 @@ decltype(auto) towards_oh_ch_regular(Pos&& p, Visitor v, size_t idx, constexpr auto B = bits; constexpr auto BL = bits_leaf; assert(offset_hint == p.index(idx)); - assert(count_hint == p.count()); + assert(count_hint == p.count()); auto is_leaf = p.shift() == BL; - auto child = p.node()->inner() [offset_hint]; + auto child = p.node()->inner()[offset_hint]; auto is_full = offset_hint + 1 != count_hint; return is_full - ? (is_leaf - ? make_full_leaf_pos(child).visit(v, idx, args...) - : make_full_pos(child, p.shift() - B).visit(v, idx, args...)) - : (is_leaf - ? make_leaf_pos(child, p.size()).visit(v, idx, args...) - : make_regular_pos(child, p.shift() - B, p.size()).visit(v, idx, args...)); + ? (is_leaf ? make_full_leaf_pos(child).visit(v, idx, args...) + : make_full_pos(child, p.shift() - B) + .visit(v, idx, args...)) + : (is_leaf + ? make_leaf_pos(child, p.size()).visit(v, idx, args...) + : make_regular_pos(child, p.shift() - B, p.size()) + .visit(v, idx, args...)); } template -decltype(auto) towards_sub_oh_regular(Pos&& p, Visitor v, size_t idx, - count_t offset_hint, - Args&&... args) +decltype(auto) towards_sub_oh_regular( + Pos&& p, Visitor v, size_t idx, count_t offset_hint, Args&&... args) { constexpr auto B = bits; constexpr auto BL = bits_leaf; assert(offset_hint == p.index(idx)); auto is_leaf = p.shift() == BL; - auto child = p.node()->inner() [offset_hint]; + auto child = p.node()->inner()[offset_hint]; auto lsize = offset_hint << p.shift(); auto size = p.this_size(); auto is_full = (size - lsize) >= (size_t{1} << p.shift()); return is_full - ? (is_leaf - ? make_full_leaf_pos(child).visit( - v, idx - lsize, args...) - : make_full_pos(child, p.shift() - B).visit( - v, idx - lsize, args...)) - : (is_leaf - ? make_leaf_sub_pos(child, size - lsize).visit( - v, idx - lsize, args...) - : make_regular_sub_pos(child, p.shift() - B, size - lsize).visit( - v, idx - lsize, args...)); + ? (is_leaf + ? make_full_leaf_pos(child).visit(v, idx - lsize, args...) + : make_full_pos(child, p.shift() - B) + .visit(v, idx - lsize, args...)) + : (is_leaf + ? make_leaf_sub_pos(child, size - lsize) + .visit(v, idx - lsize, args...) + : make_regular_sub_pos(child, p.shift() - B, size - lsize) + .visit(v, idx - lsize, args...)); } template -decltype(auto) last_oh_regular(Pos&& p, Visitor v, - count_t offset_hint, - Args&&... args) +decltype(auto) +last_oh_regular(Pos&& p, Visitor v, count_t offset_hint, Args&&... args) { assert(offset_hint == p.count() - 1); constexpr auto B = bits; constexpr auto BL = bits_leaf; - auto child = p.node()->inner() [offset_hint]; - auto is_leaf = p.shift() == BL; - return is_leaf - ? make_leaf_pos(child, p.size()).visit(v, args...) - : make_regular_pos(child, p.shift() - B, p.size()).visit(v, args...); + auto child = p.node()->inner()[offset_hint]; + auto is_leaf = p.shift() == BL; + return is_leaf ? make_leaf_pos(child, p.size()).visit(v, args...) + : make_regular_pos(child, p.shift() - B, p.size()) + .visit(v, args...); } template -regular_pos make_regular_pos(NodeT* node, - shift_t shift, - size_t size) +regular_pos make_regular_pos(NodeT* node, shift_t shift, size_t size) { assert(node); assert(shift >= NodeT::bits_leaf); @@ -657,13 +697,17 @@ struct null_sub_pos auto node() const { return nullptr; } template - void each_sub(Visitor, Args&&...) {} + void each_sub(Visitor, Args&&...) + {} template - void each_right_sub(Visitor, Args&&...) {} + void each_right_sub(Visitor, Args&&...) + {} template - void each_left_sub(Visitor, Args&&...) {} + void each_left_sub(Visitor, Args&&...) + {} template - void visit(Visitor, Args&&...) {} + void visit(Visitor, Args&&...) + {} }; template @@ -680,19 +724,21 @@ struct singleton_regular_sub_pos count_t count_; count_t count() const { return 1; } - node_t* node() const { return nullptr; } - size_t size() const { return count_; } + node_t* node() const { return nullptr; } + size_t size() const { return count_; } shift_t shift() const { return BL; } count_t index(size_t idx) const { return 0; } count_t subindex(size_t idx) const { return 0; } - size_t size_before(count_t offset) const { return 0; } - size_t this_size() const { return count_; } - size_t size(count_t offset) { return count_; } + size_t size_before(count_t offset) const { return 0; } + size_t this_size() const { return count_; } + size_t size(count_t offset) { return count_; } template - void each_left_sub(Visitor v, Args&&... args) {} + void each_left_sub(Visitor v, Args&&... args) + {} template - void each(Visitor v, Args&&... args) {} + void each(Visitor v, Args&&... args) + {} template decltype(auto) last_sub(Visitor v, Args&&... args) @@ -700,8 +746,8 @@ struct singleton_regular_sub_pos return make_leaf_sub_pos(leaf_, count_).visit(v, args...); } - template - decltype(auto) visit(Visitor v, Args&& ...args) + template + decltype(auto) visit(Visitor v, Args&&... args) { return Visitor::visit_regular(*this, std::forward(args)...); } @@ -711,7 +757,7 @@ template auto make_singleton_regular_sub_pos(NodeT* leaf, count_t count) { assert(leaf); - assert(leaf->kind() == NodeT::kind_t::leaf); + IMMER_ASSERT_TAGGED(leaf->kind() == NodeT::kind_t::leaf); assert(count > 0); return singleton_regular_sub_pos{leaf, count}; } @@ -728,86 +774,105 @@ struct regular_sub_pos size_t size_; count_t count() const { return subindex(size_ - 1) + 1; } - node_t* node() const { return node_; } - size_t size() const { return size_; } + node_t* node() const { return node_; } + size_t size() const { return size_; } shift_t shift() const { return shift_; } count_t index(size_t idx) const { return (idx >> shift_) & mask; } count_t subindex(size_t idx) const { return idx >> shift_; } - size_t size_before(count_t offset) const { return offset << shift_; } - size_t this_size() const { return size_; } + size_t size_before(count_t offset) const + { + return size_t{offset} << shift_; + } + size_t this_size() const { return size_; } auto size(count_t offset) { - return offset == subindex(size_ - 1) - ? size_ - size_before(offset) - : 1 << shift_; + return offset == subindex(size_ - 1) ? size_ - size_before(offset) + : size_t{1} << shift_; } auto size_sbh(count_t offset, size_t size_before_hint) { assert(size_before_hint == size_before(offset)); - return offset == subindex(size_ - 1) - ? size_ - size_before_hint - : 1 << shift_; + return offset == subindex(size_ - 1) ? size_ - size_before_hint + : size_t{1} << shift_; } - void copy_sizes(count_t offset, - count_t n, - size_t init, - size_t* sizes) + void copy_sizes(count_t offset, count_t n, size_t init, size_t* sizes) { if (n) { auto last = offset + n - 1; - auto e = sizes + n - 1; - for (; sizes != e; ++sizes) - init = *sizes = init + (1 << shift_); + auto e = sizes + n - 1; + for (; sizes != e; ++sizes) { + init = *sizes = init + (size_t{1} << shift_); + assert(init); + } *sizes = init + size(last); + assert(*sizes); } } template - void each(Visitor v, Args&& ...args) - { return each_regular(*this, v, args...); } + void each(Visitor v, Args&&... args) + { + return each_regular(*this, v, args...); + } template - bool each_pred(Visitor v, Args&& ...args) - { return each_pred_regular(*this, v, args...); } + bool each_pred(Visitor v, Args&&... args) + { + return each_pred_regular(*this, v, args...); + } template bool each_pred_zip(Visitor v, node_t* other, Args&&... args) - { return each_pred_zip_regular(*this, v, other, args...); } + { + return each_pred_zip_regular(*this, v, other, args...); + } template - bool each_pred_i(Visitor v, count_t i, count_t n, Args&& ...args) - { return each_pred_i_regular(*this, v, i, n, args...); } + bool each_pred_i(Visitor v, count_t i, count_t n, Args&&... args) + { + return each_pred_i_regular(*this, v, i, n, args...); + } template - bool each_pred_right(Visitor v, count_t start, Args&& ...args) - { return each_pred_right_regular(*this, v, start, args...); } + bool each_pred_right(Visitor v, count_t start, Args&&... args) + { + return each_pred_right_regular(*this, v, start, args...); + } template - bool each_pred_left(Visitor v, count_t last, Args&& ...args) - { return each_pred_left_regular(*this, v, last, args...); } + bool each_pred_left(Visitor v, count_t last, Args&&... args) + { + return each_pred_left_regular(*this, v, last, args...); + } template - void each_i(Visitor v, count_t i, count_t n, Args&& ...args) - { return each_i_regular(*this, v, i, n, args...); } + void each_i(Visitor v, count_t i, count_t n, Args&&... args) + { + return each_i_regular(*this, v, i, n, args...); + } template - void each_right(Visitor v, count_t start, Args&& ...args) - { return each_right_regular(*this, v, start, args...); } + void each_right(Visitor v, count_t start, Args&&... args) + { + return each_right_regular(*this, v, start, args...); + } template - void each_left(Visitor v, count_t last, Args&& ...args) - { return each_left_regular(*this, v, last, args...); } + void each_left(Visitor v, count_t last, Args&&... args) + { + return each_left_regular(*this, v, last, args...); + } template - void each_right_sub_(Visitor v, count_t i, Args&& ...args) + void each_right_sub_(Visitor v, count_t i, Args&&... args) { auto last = count() - 1; auto lsize = size_ - (last << shift_); - auto n = node()->inner() + i; - auto e = node()->inner() + last; + auto n = node()->inner() + i; + auto e = node()->inner() + last; if (shift() == BL) { for (; n != e; ++n) { IMMER_PREFETCH(n + 1); @@ -823,91 +888,110 @@ struct regular_sub_pos } template - void each_sub(Visitor v, Args&& ...args) - { each_right_sub_(v, 0, args...); } + void each_sub(Visitor v, Args&&... args) + { + each_right_sub_(v, 0, args...); + } template - void each_right_sub(Visitor v, Args&& ...args) - { if (count() > 1) each_right_sub_(v, 1, args...); } + void each_right_sub(Visitor v, Args&&... args) + { + if (count() > 1) + each_right_sub_(v, 1, args...); + } template - void each_left_sub(Visitor v, Args&& ...args) - { each_left(v, count() - 1, args...); } + void each_left_sub(Visitor v, Args&&... args) + { + each_left(v, count() - 1, args...); + } template decltype(auto) towards(Visitor v, size_t idx, Args&&... args) - { return towards_oh_ch_regular(*this, v, idx, index(idx), count(), args...); } + { + return towards_oh_ch_regular( + *this, v, idx, index(idx), count(), args...); + } template - decltype(auto) towards_oh(Visitor v, size_t idx, - count_t offset_hint, - Args&&... args) - { return towards_oh_ch_regular(*this, v, idx, offset_hint, count(), args...); } + decltype(auto) + towards_oh(Visitor v, size_t idx, count_t offset_hint, Args&&... args) + { + return towards_oh_ch_regular( + *this, v, idx, offset_hint, count(), args...); + } template - decltype(auto) towards_oh_ch(Visitor v, size_t idx, + decltype(auto) towards_oh_ch(Visitor v, + size_t idx, count_t offset_hint, count_t count_hint, Args&&... args) - { return towards_oh_ch_regular(*this, v, idx, offset_hint, count(), args...); } + { + return towards_oh_ch_regular( + *this, v, idx, offset_hint, count(), args...); + } template - decltype(auto) towards_sub_oh(Visitor v, size_t idx, - count_t offset_hint, - Args&& ...args) - { return towards_sub_oh_regular(*this, v, idx, offset_hint, args...); } + decltype(auto) + towards_sub_oh(Visitor v, size_t idx, count_t offset_hint, Args&&... args) + { + return towards_sub_oh_regular(*this, v, idx, offset_hint, args...); + } template decltype(auto) last_oh(Visitor v, count_t offset_hint, Args&&... args) - { return last_oh_regular(*this, v, offset_hint, args...); } + { + return last_oh_regular(*this, v, offset_hint, args...); + } template decltype(auto) last_sub(Visitor v, Args&&... args) { - auto offset = count() - 1; - auto child = node_->inner() [offset]; - auto is_leaf = shift_ == BL; - auto lsize = size_ - (offset << shift_); - return is_leaf - ? make_leaf_sub_pos(child, lsize).visit(v, args...) - : make_regular_sub_pos(child, shift_ - B, lsize).visit(v, args...); + auto offset = count() - 1; + auto child = node_->inner()[offset]; + auto is_leaf = shift_ == BL; + auto lsize = size_ - (size_t{offset} << shift_); + return is_leaf ? make_leaf_sub_pos(child, lsize).visit(v, args...) + : make_regular_sub_pos(child, shift_ - B, lsize) + .visit(v, args...); } template decltype(auto) first_sub(Visitor v, Args&&... args) { auto is_leaf = shift_ == BL; - auto child = node_->inner() [0]; + auto child = node_->inner()[0]; auto is_full = size_ >= (size_t{1} << shift_); return is_full - ? (is_leaf - ? make_full_leaf_pos(child).visit(v, args...) - : make_full_pos(child, shift_ - B).visit(v, args...)) - : (is_leaf - ? make_leaf_sub_pos(child, size_).visit(v, args...) - : make_regular_sub_pos(child, shift_ - B, size_).visit(v, args...)); + ? (is_leaf + ? make_full_leaf_pos(child).visit(v, args...) + : make_full_pos(child, shift_ - B).visit(v, args...)) + : (is_leaf + ? make_leaf_sub_pos(child, size_).visit(v, args...) + : make_regular_sub_pos(child, shift_ - B, size_) + .visit(v, args...)); } template decltype(auto) first_sub_leaf(Visitor v, Args&&... args) { assert(shift_ == BL); - auto child = node_->inner() [0]; + auto child = node_->inner()[0]; auto is_full = size_ >= branches; - return is_full - ? make_full_leaf_pos(child).visit(v, args...) - : make_leaf_sub_pos(child, size_).visit(v, args...); + return is_full ? make_full_leaf_pos(child).visit(v, args...) + : make_leaf_sub_pos(child, size_).visit(v, args...); } template decltype(auto) first_sub_inner(Visitor v, Args&&... args) { assert(shift_ >= BL); - auto child = node_->inner() [0]; + auto child = node_->inner()[0]; auto is_full = size_ >= branches; - return is_full - ? make_full_pos(child, shift_ - B).visit(v, args...) - : make_regular_sub_pos(child, shift_ - B, size_).visit(v, args...); + return is_full ? make_full_pos(child, shift_ - B).visit(v, args...) + : make_regular_sub_pos(child, shift_ - B, size_) + .visit(v, args...); } template @@ -915,41 +999,40 @@ struct regular_sub_pos { assert(idx < count()); auto is_leaf = shift_ == BL; - auto child = node_->inner() [idx]; + auto child = node_->inner()[idx]; auto lsize = size(idx); auto is_full = idx + 1 < count(); return is_full - ? (is_leaf - ? make_full_leaf_pos(child).visit(v, args...) - : make_full_pos(child, shift_ - B).visit(v, args...)) - : (is_leaf - ? make_leaf_sub_pos(child, lsize).visit(v, args...) - : make_regular_sub_pos(child, shift_ - B, lsize).visit(v, args...)); + ? (is_leaf + ? make_full_leaf_pos(child).visit(v, args...) + : make_full_pos(child, shift_ - B).visit(v, args...)) + : (is_leaf + ? make_leaf_sub_pos(child, lsize).visit(v, args...) + : make_regular_sub_pos(child, shift_ - B, lsize) + .visit(v, args...)); } template decltype(auto) nth_sub_leaf(count_t idx, Visitor v, Args&&... args) { assert(shift_ == BL); - auto child = node_->inner() [idx]; + auto child = node_->inner()[idx]; auto lsize = size(idx); auto is_full = idx + 1 < count(); - return is_full - ? make_full_leaf_pos(child).visit(v, args...) - : make_leaf_sub_pos(child, lsize).visit(v, args...); + return is_full ? make_full_leaf_pos(child).visit(v, args...) + : make_leaf_sub_pos(child, lsize).visit(v, args...); } - template - decltype(auto) visit(Visitor v, Args&& ...args) + template + decltype(auto) visit(Visitor v, Args&&... args) { return Visitor::visit_regular(*this, std::forward(args)...); } }; template -regular_sub_pos make_regular_sub_pos(NodeT* node, - shift_t shift, - size_t size) +regular_sub_pos +make_regular_sub_pos(NodeT* node, shift_t shift, size_t size) { assert(node); assert(shift >= NodeT::bits_leaf); @@ -958,7 +1041,8 @@ regular_sub_pos make_regular_sub_pos(NodeT* node, return {node, shift, size}; } -template struct regular_descent_pos @@ -968,9 +1052,10 @@ struct regular_descent_pos using node_t = NodeT; node_t* node_; - node_t* node() const { return node_; } + node_t* node() const { return node_; } shift_t shift() const { return Shift; } - count_t index(size_t idx) const { + count_t index(size_t idx) const + { #if !defined(_MSC_VER) #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wshift-count-overflow" @@ -989,8 +1074,8 @@ struct regular_descent_pos return regular_descent_pos{child}.visit(v, idx); } - template - decltype(auto) visit(Visitor v, Args&& ...args) + template + decltype(auto) visit(Visitor v, Args&&... args) { return Visitor::visit_regular(*this, std::forward(args)...); } @@ -1002,7 +1087,7 @@ struct regular_descent_pos using node_t = NodeT; node_t* node_; - node_t* node() const { return node_; } + node_t* node() const { return node_; } shift_t shift() const { return BL; } count_t index(size_t idx) const { return (idx >> BL) & mask; } @@ -1014,32 +1099,38 @@ struct regular_descent_pos return make_leaf_descent_pos(child).visit(v, idx); } - template - decltype(auto) visit(Visitor v, Args&& ...args) + template + decltype(auto) visit(Visitor v, Args&&... args) { return Visitor::visit_regular(*this, std::forward(args)...); } }; template -decltype(auto) visit_regular_descent(NodeT* node, shift_t shift, Visitor v, - size_t idx) +decltype(auto) +visit_regular_descent(NodeT* node, shift_t shift, Visitor v, size_t idx) { constexpr auto B = NodeT::bits; constexpr auto BL = NodeT::bits_leaf; assert(node); assert(shift >= BL); switch (shift) { - case BL + B * 0: return regular_descent_pos{node}.visit(v, idx); - case BL + B * 1: return regular_descent_pos{node}.visit(v, idx); - case BL + B * 2: return regular_descent_pos{node}.visit(v, idx); - case BL + B * 3: return regular_descent_pos{node}.visit(v, idx); - case BL + B * 4: return regular_descent_pos{node}.visit(v, idx); - case BL + B * 5: return regular_descent_pos{node}.visit(v, idx); + case BL + B * 0: + return regular_descent_pos{node}.visit(v, idx); + case BL + B * 1: + return regular_descent_pos{node}.visit(v, idx); + case BL + B * 2: + return regular_descent_pos{node}.visit(v, idx); + case BL + B * 3: + return regular_descent_pos{node}.visit(v, idx); + case BL + B * 4: + return regular_descent_pos{node}.visit(v, idx); + case BL + B * 5: + return regular_descent_pos{node}.visit(v, idx); #if IMMER_DESCENT_DEEP default: for (auto level = shift; level != endshift; level -= B) - node = node->inner() [(idx >> level) & mask]; + node = node->inner()[(idx >> level) & mask]; return make_leaf_descent_pos(node).visit(v, idx); #endif // IMMER_DEEP_DESCENT } @@ -1057,23 +1148,28 @@ struct full_pos shift_t shift_; count_t count() const { return branches; } - node_t* node() const { return node_; } - size_t size() const { return branches << shift_; } + node_t* node() const { return node_; } + size_t size() const { return branches << shift_; } shift_t shift() const { return shift_; } count_t index(size_t idx) const { return (idx >> shift_) & mask; } count_t subindex(size_t idx) const { return idx >> shift_; } - size_t size(count_t offset) const { return 1 << shift_; } - size_t size_sbh(count_t offset, size_t) const { return 1 << shift_; } - size_t size_before(count_t offset) const { return offset << shift_; } + size_t size(count_t offset) const { return size_t{1} << shift_; } + size_t size_sbh(count_t offset, size_t) const + { + return size_t{1} << shift_; + } + size_t size_before(count_t offset) const + { + return size_t{offset} << shift_; + } - void copy_sizes(count_t offset, - count_t n, - size_t init, - size_t* sizes) + void copy_sizes(count_t offset, count_t n, size_t init, size_t* sizes) { auto e = sizes + n; - for (; sizes != e; ++sizes) - init = *sizes = init + (1 << shift_); + for (; sizes != e; ++sizes) { + init = *sizes = init + (size_t{1} << shift_); + assert(init); + } } template @@ -1116,9 +1212,9 @@ struct full_pos template bool each_pred_zip(Visitor v, node_t* other, Args&&... args) { - auto p = node_->inner(); + auto p = node_->inner(); auto p2 = other->inner(); - auto e = p + branches; + auto e = p + branches; if (shift_ == BL) { for (; p != e; ++p, ++p2) { IMMER_PREFETCH(p + 1); @@ -1173,84 +1269,99 @@ struct full_pos template bool each_pred_right(Visitor v, count_t start, Args&&... args) - { return each_pred_i(v, start, branches, args...); } + { + return each_pred_i(v, start, branches, args...); + } template bool each_pred_left(Visitor v, count_t last, Args&&... args) - { return each_pred_i(v, 0, last, args...); } + { + return each_pred_i(v, 0, last, args...); + } template void each_sub(Visitor v, Args&&... args) - { each(v, args...); } + { + each(v, args...); + } template void each_left_sub(Visitor v, Args&&... args) - { each_i(v, 0, branches - 1, args...); } + { + each_i(v, 0, branches - 1, args...); + } template void each_right_sub(Visitor v, Args&&... args) - { each_i(v, 1, branches, args...); } + { + each_i(v, 1, branches, args...); + } template void each_right(Visitor v, count_t start, Args&&... args) - { each_i(v, start, branches, args...); } + { + each_i(v, start, branches, args...); + } template void each_left(Visitor v, count_t last, Args&&... args) - { each_i(v, 0, last, args...); } + { + each_i(v, 0, last, args...); + } template decltype(auto) towards(Visitor v, size_t idx, Args&&... args) - { return towards_oh(v, idx, index(idx), args...); } + { + return towards_oh(v, idx, index(idx), args...); + } template - decltype(auto) towards_oh_ch(Visitor v, size_t idx, - count_t offset_hint, count_t, - Args&&... args) - { return towards_oh(v, idx, offset_hint, args...); } + decltype(auto) towards_oh_ch( + Visitor v, size_t idx, count_t offset_hint, count_t, Args&&... args) + { + return towards_oh(v, idx, offset_hint, args...); + } template - decltype(auto) towards_oh(Visitor v, size_t idx, - count_t offset_hint, - Args&&... args) + decltype(auto) + towards_oh(Visitor v, size_t idx, count_t offset_hint, Args&&... args) { assert(offset_hint == index(idx)); auto is_leaf = shift_ == BL; - auto child = node_->inner() [offset_hint]; + auto child = node_->inner()[offset_hint]; return is_leaf - ? make_full_leaf_pos(child).visit(v, idx, args...) - : make_full_pos(child, shift_ - B).visit(v, idx, args...); + ? make_full_leaf_pos(child).visit(v, idx, args...) + : make_full_pos(child, shift_ - B).visit(v, idx, args...); } template - decltype(auto) towards_sub_oh(Visitor v, size_t idx, - count_t offset_hint, - Args&&... args) + decltype(auto) + towards_sub_oh(Visitor v, size_t idx, count_t offset_hint, Args&&... args) { assert(offset_hint == index(idx)); auto is_leaf = shift_ == BL; - auto child = node_->inner() [offset_hint]; - auto lsize = offset_hint << shift_; + auto child = node_->inner()[offset_hint]; + auto lsize = size_t{offset_hint} << shift_; return is_leaf - ? make_full_leaf_pos(child).visit(v, idx - lsize, args...) - : make_full_pos(child, shift_ - B).visit(v, idx - lsize, args...); + ? make_full_leaf_pos(child).visit(v, idx - lsize, args...) + : make_full_pos(child, shift_ - B) + .visit(v, idx - lsize, args...); } template decltype(auto) first_sub(Visitor v, Args&&... args) { auto is_leaf = shift_ == BL; - auto child = node_->inner() [0]; - return is_leaf - ? make_full_leaf_pos(child).visit(v, args...) - : make_full_pos(child, shift_ - B).visit(v, args...); + auto child = node_->inner()[0]; + return is_leaf ? make_full_leaf_pos(child).visit(v, args...) + : make_full_pos(child, shift_ - B).visit(v, args...); } template decltype(auto) first_sub_leaf(Visitor v, Args&&... args) { assert(shift_ == BL); - auto child = node_->inner() [0]; + auto child = node_->inner()[0]; return make_full_leaf_pos(child).visit(v, args...); } @@ -1258,7 +1369,7 @@ struct full_pos decltype(auto) first_sub_inner(Visitor v, Args&&... args) { assert(shift_ >= BL); - auto child = node_->inner() [0]; + auto child = node_->inner()[0]; return make_full_pos(child, shift_ - B).visit(v, args...); } @@ -1267,10 +1378,9 @@ struct full_pos { assert(idx < count()); auto is_leaf = shift_ == BL; - auto child = node_->inner() [idx]; - return is_leaf - ? make_full_leaf_pos(child).visit(v, args...) - : make_full_pos(child, shift_ - B).visit(v, args...); + auto child = node_->inner()[idx]; + return is_leaf ? make_full_leaf_pos(child).visit(v, args...) + : make_full_pos(child, shift_ - B).visit(v, args...); } template @@ -1278,12 +1388,12 @@ struct full_pos { assert(shift_ == BL); assert(idx < count()); - auto child = node_->inner() [idx]; + auto child = node_->inner()[idx]; return make_full_leaf_pos(child).visit(v, args...); } - template - decltype(auto) visit(Visitor v, Args&& ...args) + template + decltype(auto) visit(Visitor v, Args&&... args) { return Visitor::visit_regular(*this, std::forward(args)...); } @@ -1303,24 +1413,28 @@ struct relaxed_pos static constexpr auto B = NodeT::bits; static constexpr auto BL = NodeT::bits_leaf; - using node_t = NodeT; + using node_t = NodeT; using relaxed_t = typename NodeT::relaxed_t; node_t* node_; shift_t shift_; relaxed_t* relaxed_; count_t count() const { return relaxed_->d.count; } - node_t* node() const { return node_; } - size_t size() const { return relaxed_->d.sizes[relaxed_->d.count - 1]; } + node_t* node() const { return node_; } + size_t size() const { return relaxed_->d.sizes[relaxed_->d.count - 1]; } shift_t shift() const { return shift_; } count_t subindex(size_t idx) const { return index(idx); } relaxed_t* relaxed() const { return relaxed_; } size_t size_before(count_t offset) const - { return offset ? relaxed_->d.sizes[offset - 1] : 0; } + { + return offset ? relaxed_->d.sizes[offset - 1] : 0; + } size_t size(count_t offset) const - { return size_sbh(offset, size_before(offset)); } + { + return size_sbh(offset, size_before(offset)); + } size_t size_sbh(count_t offset, size_t size_before_hint) const { @@ -1331,28 +1445,29 @@ struct relaxed_pos count_t index(size_t idx) const { auto offset = idx >> shift_; - while (relaxed_->d.sizes[offset] <= idx) ++offset; + while (relaxed_->d.sizes[offset] <= idx) + ++offset; return offset; } - void copy_sizes(count_t offset, - count_t n, - size_t init, - size_t* sizes) + void copy_sizes(count_t offset, count_t n, size_t init, size_t* sizes) { - auto e = sizes + n; - auto prev = size_before(offset); + auto e = sizes + n; + auto prev = size_before(offset); auto these = relaxed_->d.sizes + offset; for (; sizes != e; ++sizes, ++these) { auto this_size = *these; init = *sizes = init + (this_size - prev); + assert(init); prev = this_size; } } template void each(Visitor v, Args&&... args) - { each_left(v, relaxed_->d.count, args...); } + { + each_left(v, relaxed_->d.count, args...); + } template bool each_pred(Visitor v, Args&&... args) @@ -1364,15 +1479,15 @@ struct relaxed_pos for (auto i = count_t{0}; i < n; ++i) { IMMER_PREFETCH(p + i + 1); if (!make_leaf_sub_pos(p[i], relaxed_->d.sizes[i] - s) - .visit(v, args...)) + .visit(v, args...)) return false; s = relaxed_->d.sizes[i]; } } else { auto ss = shift_ - B; for (auto i = count_t{0}; i < n; ++i) { - if (!visit_maybe_relaxed_sub(p[i], ss, relaxed_->d.sizes[i] - s, - v, args...)) + if (!visit_maybe_relaxed_sub( + p[i], ss, relaxed_->d.sizes[i] - s, v, args...)) return false; s = relaxed_->d.sizes[i]; } @@ -1389,17 +1504,17 @@ struct relaxed_pos for (; i < n; ++i) { IMMER_PREFETCH(p + i + 1); if (!make_leaf_sub_pos(p[i], relaxed_->d.sizes[i] - s) - .visit(v, args...)) + .visit(v, args...)) return false; s = relaxed_->d.sizes[i]; } } else { - auto p = node_->inner(); - auto s = i > 0 ? relaxed_->d.sizes[i - 1] : 0; + auto p = node_->inner(); + auto s = i > 0 ? relaxed_->d.sizes[i - 1] : 0; auto ss = shift_ - B; for (; i < n; ++i) { - if (!visit_maybe_relaxed_sub(p[i], ss, relaxed_->d.sizes[i] - s, - v, args...)) + if (!visit_maybe_relaxed_sub( + p[i], ss, relaxed_->d.sizes[i] - s, v, args...)) return false; s = relaxed_->d.sizes[i]; } @@ -1416,15 +1531,15 @@ struct relaxed_pos for (auto i = count_t{0}; i < n; ++i) { IMMER_PREFETCH(p + i + 1); if (!make_leaf_sub_pos(p[i], relaxed_->d.sizes[i] - s) - .visit(v, args...)) + .visit(v, args...)) return false; s = relaxed_->d.sizes[i]; } } else { auto ss = shift_ - B; for (auto i = count_t{0}; i < n; ++i) { - if (!visit_maybe_relaxed_sub(p[i], ss, relaxed_->d.sizes[i] - s, - v, args...)) + if (!visit_maybe_relaxed_sub( + p[i], ss, relaxed_->d.sizes[i] - s, v, args...)) return false; s = relaxed_->d.sizes[i]; } @@ -1443,15 +1558,15 @@ struct relaxed_pos for (auto i = start; i < relaxed_->d.count; ++i) { IMMER_PREFETCH(p + i + 1); if (!make_leaf_sub_pos(p[i], relaxed_->d.sizes[i] - s) - .visit(v, args...)) + .visit(v, args...)) return false; s = relaxed_->d.sizes[i]; } } else { auto ss = shift_ - B; for (auto i = start; i < relaxed_->d.count; ++i) { - if (!visit_maybe_relaxed_sub(p[i], ss, relaxed_->d.sizes[i] - s, - v, args...)) + if (!visit_maybe_relaxed_sub( + p[i], ss, relaxed_->d.sizes[i] - s, v, args...)) return false; s = relaxed_->d.sizes[i]; } @@ -1472,12 +1587,12 @@ struct relaxed_pos s = relaxed_->d.sizes[i]; } } else { - auto p = node_->inner(); - auto s = i > 0 ? relaxed_->d.sizes[i - 1] : 0; + auto p = node_->inner(); + auto s = i > 0 ? relaxed_->d.sizes[i - 1] : 0; auto ss = shift_ - B; for (; i < n; ++i) { - visit_maybe_relaxed_sub(p[i], ss, relaxed_->d.sizes[i] - s, - v, args...); + visit_maybe_relaxed_sub( + p[i], ss, relaxed_->d.sizes[i] - s, v, args...); s = relaxed_->d.sizes[i]; } } @@ -1485,11 +1600,15 @@ struct relaxed_pos template void each_sub(Visitor v, Args&&... args) - { each_left(v, relaxed_->d.count, args...); } + { + each_left(v, relaxed_->d.count, args...); + } template void each_left_sub(Visitor v, Args&&... args) - { each_left(v, relaxed_->d.count - 1, args...); } + { + each_left(v, relaxed_->d.count - 1, args...); + } template void each_left(Visitor v, count_t n, Args&&... args) @@ -1502,20 +1621,24 @@ struct relaxed_pos make_leaf_sub_pos(p[i], relaxed_->d.sizes[i] - s) .visit(v, args...); s = relaxed_->d.sizes[i]; + assert(s); } } else { auto ss = shift_ - B; for (auto i = count_t{0}; i < n; ++i) { - visit_maybe_relaxed_sub(p[i], ss, relaxed_->d.sizes[i] - s, - v, args...); + visit_maybe_relaxed_sub( + p[i], ss, relaxed_->d.sizes[i] - s, v, args...); s = relaxed_->d.sizes[i]; + assert(s); } } } template void each_right_sub(Visitor v, Args&&... args) - { each_right(v, 1, std::forward(args)...); } + { + each_right(v, 1, std::forward(args)...); + } template void each_right(Visitor v, count_t start, Args&&... args) @@ -1530,25 +1653,28 @@ struct relaxed_pos make_leaf_sub_pos(p[i], relaxed_->d.sizes[i] - s) .visit(v, args...); s = relaxed_->d.sizes[i]; + assert(s); } } else { auto ss = shift_ - B; for (auto i = start; i < relaxed_->d.count; ++i) { - visit_maybe_relaxed_sub(p[i], ss, relaxed_->d.sizes[i] - s, - v, args...); + visit_maybe_relaxed_sub( + p[i], ss, relaxed_->d.sizes[i] - s, v, args...); s = relaxed_->d.sizes[i]; + assert(s); } } } template decltype(auto) towards(Visitor v, size_t idx, Args&&... args) - { return towards_oh(v, idx, subindex(idx), args...); } + { + return towards_oh(v, idx, subindex(idx), args...); + } template - decltype(auto) towards_oh(Visitor v, size_t idx, - count_t offset_hint, - Args&&... args) + decltype(auto) + towards_oh(Visitor v, size_t idx, count_t offset_hint, Args&&... args) { assert(offset_hint == index(idx)); auto left_size = offset_hint ? relaxed_->d.sizes[offset_hint - 1] : 0; @@ -1556,16 +1682,18 @@ struct relaxed_pos } template - decltype(auto) towards_oh_sbh(Visitor v, size_t idx, + decltype(auto) towards_oh_sbh(Visitor v, + size_t idx, count_t offset_hint, size_t left_size_hint, Args&&... args) - { return towards_sub_oh_sbh(v, idx, offset_hint, left_size_hint, args...); } + { + return towards_sub_oh_sbh(v, idx, offset_hint, left_size_hint, args...); + } template - decltype(auto) towards_sub_oh(Visitor v, size_t idx, - count_t offset_hint, - Args&&... args) + decltype(auto) + towards_sub_oh(Visitor v, size_t idx, count_t offset_hint, Args&&... args) { assert(offset_hint == index(idx)); auto left_size = offset_hint ? relaxed_->d.sizes[offset_hint - 1] : 0; @@ -1573,7 +1701,8 @@ struct relaxed_pos } template - decltype(auto) towards_sub_oh_sbh(Visitor v, size_t idx, + decltype(auto) towards_sub_oh_sbh(Visitor v, + size_t idx, count_t offset_hint, size_t left_size_hint, Args&&... args) @@ -1581,15 +1710,15 @@ struct relaxed_pos assert(offset_hint == index(idx)); assert(left_size_hint == (offset_hint ? relaxed_->d.sizes[offset_hint - 1] : 0)); - auto child = node_->inner() [offset_hint]; + auto child = node_->inner()[offset_hint]; auto is_leaf = shift_ == BL; auto next_size = relaxed_->d.sizes[offset_hint] - left_size_hint; auto next_idx = idx - left_size_hint; return is_leaf - ? make_leaf_sub_pos(child, next_size).visit( - v, next_idx, args...) - : visit_maybe_relaxed_sub(child, shift_ - B, next_size, - v, next_idx, args...); + ? make_leaf_sub_pos(child, next_size) + .visit(v, next_idx, args...) + : visit_maybe_relaxed_sub( + child, shift_ - B, next_size, v, next_idx, args...); } template @@ -1600,42 +1729,43 @@ struct relaxed_pos { assert(offset_hint == count() - 1); assert(child_size_hint == size(offset_hint)); - auto child = node_->inner() [offset_hint]; - auto is_leaf = shift_ == BL; + auto child = node_->inner()[offset_hint]; + auto is_leaf = shift_ == BL; return is_leaf - ? make_leaf_sub_pos(child, child_size_hint).visit(v, args...) - : visit_maybe_relaxed_sub(child, shift_ - B, child_size_hint, - v, args...); + ? make_leaf_sub_pos(child, child_size_hint).visit(v, args...) + : visit_maybe_relaxed_sub( + child, shift_ - B, child_size_hint, v, args...); } template decltype(auto) last_sub(Visitor v, Args&&... args) { auto offset = relaxed_->d.count - 1; - auto child = node_->inner() [offset]; + auto child = node_->inner()[offset]; auto child_size = size(offset); auto is_leaf = shift_ == BL; - return is_leaf - ? make_leaf_sub_pos(child, child_size).visit(v, args...) - : visit_maybe_relaxed_sub(child, shift_ - B, child_size, v, args...); + return is_leaf ? make_leaf_sub_pos(child, child_size).visit(v, args...) + : visit_maybe_relaxed_sub( + child, shift_ - B, child_size, v, args...); } template decltype(auto) first_sub(Visitor v, Args&&... args) { - auto child = node_->inner() [0]; + auto child = node_->inner()[0]; auto child_size = relaxed_->d.sizes[0]; auto is_leaf = shift_ == BL; - return is_leaf - ? make_leaf_sub_pos(child, child_size).visit(v, args...) - : visit_maybe_relaxed_sub(child, shift_ - B, child_size, v, args...); + assert(child_size); + return is_leaf ? make_leaf_sub_pos(child, child_size).visit(v, args...) + : visit_maybe_relaxed_sub( + child, shift_ - B, child_size, v, args...); } template decltype(auto) first_sub_leaf(Visitor v, Args&&... args) { assert(shift_ == BL); - auto child = node_->inner() [0]; + auto child = node_->inner()[0]; auto child_size = relaxed_->d.sizes[0]; return make_leaf_sub_pos(child, child_size).visit(v, args...); } @@ -1644,33 +1774,34 @@ struct relaxed_pos decltype(auto) first_sub_inner(Visitor v, Args&&... args) { assert(shift_ > BL); - auto child = node_->inner() [0]; + auto child = node_->inner()[0]; auto child_size = relaxed_->d.sizes[0]; - return visit_maybe_relaxed_sub(child, shift_ - B, child_size, v, args...); + return visit_maybe_relaxed_sub( + child, shift_ - B, child_size, v, args...); } template decltype(auto) nth_sub(count_t offset, Visitor v, Args&&... args) { - auto child = node_->inner() [offset]; + auto child = node_->inner()[offset]; auto child_size = size(offset); auto is_leaf = shift_ == BL; - return is_leaf - ? make_leaf_sub_pos(child, child_size).visit(v, args...) - : visit_maybe_relaxed_sub(child, shift_ - B, child_size, v, args...); + return is_leaf ? make_leaf_sub_pos(child, child_size).visit(v, args...) + : visit_maybe_relaxed_sub( + child, shift_ - B, child_size, v, args...); } template decltype(auto) nth_sub_leaf(count_t offset, Visitor v, Args&&... args) { assert(shift_ == BL); - auto child = node_->inner() [offset]; + auto child = node_->inner()[offset]; auto child_size = size(offset); return make_leaf_sub_pos(child, child_size).visit(v, args...); } - template - decltype(auto) visit(Visitor v, Args&& ...args) + template + decltype(auto) visit(Visitor v, Args&&... args) { return Visitor::visit_relaxed(*this, std::forward(args)...); } @@ -1684,9 +1815,8 @@ template constexpr auto is_relaxed_v = is_relaxed::value; template -relaxed_pos make_relaxed_pos(NodeT* node, - shift_t shift, - typename NodeT::relaxed_t* relaxed) +relaxed_pos +make_relaxed_pos(NodeT* node, shift_t shift, typename NodeT::relaxed_t* relaxed) { assert(node); assert(relaxed); @@ -1695,8 +1825,8 @@ relaxed_pos make_relaxed_pos(NodeT* node, } template -decltype(auto) visit_maybe_relaxed_sub(NodeT* node, shift_t shift, size_t size, - Visitor v, Args&& ...args) +decltype(auto) visit_maybe_relaxed_sub( + NodeT* node, shift_t shift, size_t size, Visitor v, Args&&... args) { assert(node); auto relaxed = node->relaxed(); @@ -1710,22 +1840,23 @@ decltype(auto) visit_maybe_relaxed_sub(NodeT* node, shift_t shift, size_t size, } } -template struct relaxed_descent_pos { static_assert(Shift > 0, "not leaf..."); - using node_t = NodeT; + using node_t = NodeT; using relaxed_t = typename NodeT::relaxed_t; node_t* node_; relaxed_t* relaxed_; count_t count() const { return relaxed_->d.count; } - node_t* node() const { return node_; } + node_t* node() const { return node_; } shift_t shift() const { return Shift; } - size_t size() const { return relaxed_->d.sizes[relaxed_->d.count - 1]; } + size_t size() const { return relaxed_->d.sizes[relaxed_->d.count - 1]; } count_t index(size_t idx) const { @@ -1738,7 +1869,8 @@ struct relaxed_descent_pos #if !defined(_MSC_VER) #pragma GCC diagnostic pop #endif - while (relaxed_->d.sizes[offset] <= idx) ++offset; + while (relaxed_->d.sizes[offset] <= idx) + ++offset; return offset; } @@ -1746,17 +1878,18 @@ struct relaxed_descent_pos decltype(auto) descend(Visitor v, size_t idx) { auto offset = index(idx); - auto child = node_->inner() [offset]; + auto child = node_->inner()[offset]; auto left_size = offset ? relaxed_->d.sizes[offset - 1] : 0; auto next_idx = idx - left_size; - auto r = child->relaxed(); - return r - ? relaxed_descent_pos{child, r}.visit(v, next_idx) - : regular_descent_pos{child}.visit(v, next_idx); + auto r = child->relaxed(); + return r ? relaxed_descent_pos{child, r}.visit( + v, next_idx) + : regular_descent_pos{child}.visit(v, + next_idx); } - template - decltype(auto) visit(Visitor v, Args&& ...args) + template + decltype(auto) visit(Visitor v, Args&&... args) { return Visitor::visit_relaxed(*this, std::forward(args)...); } @@ -1765,20 +1898,21 @@ struct relaxed_descent_pos template struct relaxed_descent_pos { - using node_t = NodeT; + using node_t = NodeT; using relaxed_t = typename NodeT::relaxed_t; node_t* node_; relaxed_t* relaxed_; count_t count() const { return relaxed_->d.count; } - node_t* node() const { return node_; } + node_t* node() const { return node_; } shift_t shift() const { return BL; } - size_t size() const { return relaxed_->d.sizes[relaxed_->d.count - 1]; } + size_t size() const { return relaxed_->d.sizes[relaxed_->d.count - 1]; } count_t index(size_t idx) const { auto offset = (idx >> BL) & mask; - while (relaxed_->d.sizes[offset] <= idx) ++offset; + while (relaxed_->d.sizes[offset] <= idx) + ++offset; return offset; } @@ -1786,22 +1920,22 @@ struct relaxed_descent_pos decltype(auto) descend(Visitor v, size_t idx) { auto offset = index(idx); - auto child = node_->inner() [offset]; + auto child = node_->inner()[offset]; auto left_size = offset ? relaxed_->d.sizes[offset - 1] : 0; auto next_idx = idx - left_size; return leaf_descent_pos{child}.visit(v, next_idx); } - template - decltype(auto) visit(Visitor v, Args&& ...args) + template + decltype(auto) visit(Visitor v, Args&&... args) { return Visitor::visit_relaxed(*this, std::forward(args)...); } }; template -decltype(auto) visit_maybe_relaxed_descent(NodeT* node, shift_t shift, - Visitor v, size_t idx) +decltype(auto) +visit_maybe_relaxed_descent(NodeT* node, shift_t shift, Visitor v, size_t idx) { constexpr auto B = NodeT::bits; constexpr auto BL = NodeT::bits_leaf; @@ -1810,24 +1944,38 @@ decltype(auto) visit_maybe_relaxed_descent(NodeT* node, shift_t shift, auto r = node->relaxed(); if (r) { switch (shift) { - case BL + B * 0: return relaxed_descent_pos{node, r}.visit(v, idx); - case BL + B * 1: return relaxed_descent_pos{node, r}.visit(v, idx); - case BL + B * 2: return relaxed_descent_pos{node, r}.visit(v, idx); - case BL + B * 3: return relaxed_descent_pos{node, r}.visit(v, idx); - case BL + B * 4: return relaxed_descent_pos{node, r}.visit(v, idx); - case BL + B * 5: return relaxed_descent_pos{node, r}.visit(v, idx); + case BL + B * 0: + return relaxed_descent_pos{node, r}.visit(v, + idx); + case BL + B * 1: + return relaxed_descent_pos{node, r}.visit(v, + idx); + case BL + B * 2: + return relaxed_descent_pos{node, r}.visit(v, + idx); + case BL + B * 3: + return relaxed_descent_pos{node, r}.visit(v, + idx); + case BL + B * 4: + return relaxed_descent_pos{node, r}.visit(v, + idx); + case BL + B * 5: + return relaxed_descent_pos{node, r}.visit(v, + idx); #if IMMER_DESCENT_DEEP default: for (auto level = shift; level != endshift; level -= B) { auto r = node->relaxed(); if (r) { auto node_idx = (idx >> level) & mask; - while (r->d.sizes[node_idx] <= idx) ++node_idx; - if (node_idx) idx -= r->d.sizes[node_idx - 1]; - node = node->inner() [node_idx]; + while (r->d.sizes[node_idx] <= idx) + ++node_idx; + if (node_idx) + idx -= r->d.sizes[node_idx - 1]; + node = node->inner()[node_idx]; } else { do { - node = node->inner() [(idx >> level) & mask]; + node = node->inner()[(idx >> level) & mask]; } while ((level -= B) != endshift); return make_leaf_descent_pos(node).visit(v, idx); } diff --git a/src/immer/detail/rbts/rbtree.hpp b/src/immer/detail/rbts/rbtree.hpp index aab5bd9fe3..c8fa4251f0 100644 --- a/src/immer/detail/rbts/rbtree.hpp +++ b/src/immer/detail/rbts/rbtree.hpp @@ -8,63 +8,68 @@ #pragma once +#include #include -#include #include - +#include #include #include #include #include +#include namespace immer { namespace detail { namespace rbts { -template +template struct rbtree { using node_t = node; using edit_t = typename node_t::edit_t; using owner_t = typename MemoryPolicy::transience_t::owner; - size_t size; - shift_t shift; - node_t* root; - node_t* tail; + size_t size; + shift_t shift; + node_t* root; + node_t* tail; + + constexpr static size_t max_size() + { + auto S = sizeof(size_t) * 8; + return (size_t{1} << BL) * ipow(size_t{1} << B, (S - BL) / B); + } + + static node_t* empty_root() + { + static const auto empty_ = node_t::make_inner_n(0u); + return empty_->inc(); + } - static const rbtree& empty() + static node_t* empty_tail() { - static const rbtree empty_ { - 0, - BL, - node_t::make_inner_n(0u), - node_t::make_leaf_n(0u) - }; - return empty_; + static const auto empty_ = node_t::make_leaf_n(0u); + return empty_->inc(); } template static auto from_initializer_list(std::initializer_list values) { - auto e = owner_t{}; - auto result = rbtree{empty()}; + auto e = owner_t{}; + auto result = rbtree{}; for (auto&& v : values) result.push_back_mut(e, v); return result; } - template , bool> = true> + template , bool> = true> static auto from_range(Iter first, Sent last) { - auto e = owner_t{}; - auto result = rbtree{empty()}; + auto e = owner_t{}; + auto result = rbtree{}; for (; first != last; ++first) result.push_back_mut(e, *first); return result; @@ -72,15 +77,27 @@ struct rbtree static auto from_fill(size_t n, T v) { - auto e = owner_t{}; - auto result = rbtree{empty()}; - while (n --> 0) + auto e = owner_t{}; + auto result = rbtree{}; + while (n-- > 0) result.push_back_mut(e, v); return result; } + rbtree() + : size{0} + , shift{BL} + , root{empty_root()} + , tail{empty_tail()} + { + assert(check_tree()); + } + rbtree(size_t sz, shift_t sh, node_t* r, node_t* t) - : size{sz}, shift{sh}, root{r}, tail{t} + : size{sz} + , shift{sh} + , root{r} + , tail{t} { assert(check_tree()); } @@ -92,7 +109,7 @@ struct rbtree } rbtree(rbtree&& other) - : rbtree{empty()} + : rbtree{} { swap(*this, other); } @@ -113,16 +130,13 @@ struct rbtree friend void swap(rbtree& x, rbtree& y) { using std::swap; - swap(x.size, y.size); + swap(x.size, y.size); swap(x.shift, y.shift); - swap(x.root, y.root); - swap(x.tail, y.tail); + swap(x.root, y.root); + swap(x.tail, y.tail); } - ~rbtree() - { - dec(); - } + ~rbtree() { dec(); } void inc() const { @@ -130,20 +144,11 @@ struct rbtree tail->inc(); } - void dec() const - { - traverse(dec_visitor()); - } + void dec() const { traverse(dec_visitor()); } - auto tail_size() const - { - return size ? ((size - 1) & mask) + 1 : 0; - } + auto tail_size() const { return size ? ((size - 1) & mask) +1 : 0; } - auto tail_offset() const - { - return size ? (size - 1) & ~mask : 0; - } + auto tail_offset() const { return size ? (size - 1) & ~mask : 0; } template void traverse(Visitor v, Args&&... args) const @@ -151,8 +156,10 @@ struct rbtree auto tail_off = tail_offset(); auto tail_size = size - tail_off; - if (tail_off) make_regular_sub_pos(root, shift, tail_off).visit(v, args...); - else make_empty_regular_pos(root).visit(v, args...); + if (tail_off) + make_regular_sub_pos(root, shift, tail_off).visit(v, args...); + else + make_empty_regular_pos(root).visit(v, args...); make_leaf_sub_pos(tail, tail_size).visit(v, args...); } @@ -164,17 +171,14 @@ struct rbtree auto tail_size = size - tail_off; if (first < tail_off) - make_regular_sub_pos(root, shift, tail_off).visit( - v, - first, - last < tail_off ? last : tail_off, - args...); + make_regular_sub_pos(root, shift, tail_off) + .visit(v, first, last < tail_off ? last : tail_off, args...); if (last > tail_off) - make_leaf_sub_pos(tail, tail_size).visit( - v, - first > tail_off ? first - tail_off : 0, - last - tail_off, - args...); + make_leaf_sub_pos(tail, tail_size) + .visit(v, + first > tail_off ? first - tail_off : 0, + last - tail_off, + args...); } template @@ -182,10 +186,10 @@ struct rbtree { auto tail_off = tail_offset(); auto tail_size = size - tail_off; - return (tail_off - ? make_regular_sub_pos(root, shift, tail_off).visit(v, args...) - : make_empty_regular_pos(root).visit(v, args...)) - && make_leaf_sub_pos(tail, tail_size).visit(v, args...); + return (tail_off ? make_regular_sub_pos(root, shift, tail_off) + .visit(v, args...) + : make_empty_regular_pos(root).visit(v, args...)) && + make_leaf_sub_pos(tail, tail_size).visit(v, args...); } template @@ -194,30 +198,27 @@ struct rbtree auto tail_off = tail_offset(); auto tail_size = size - tail_off; - return - (first < tail_off - ? make_regular_sub_pos(root, shift, tail_off).visit( - v, - first, - last < tail_off ? last : tail_off, - args...) - : true) - && (last > tail_off - ? make_leaf_sub_pos(tail, tail_size).visit( - v, - first > tail_off ? first - tail_off : 0, - last - tail_off, - args...) - : true); + return (first < tail_off ? make_regular_sub_pos(root, shift, tail_off) + .visit(v, + first, + last < tail_off ? last : tail_off, + args...) + : true) && + (last > tail_off + ? make_leaf_sub_pos(tail, tail_size) + .visit(v, + first > tail_off ? first - tail_off : 0, + last - tail_off, + args...) + : true); } template decltype(auto) descend(Visitor v, size_t idx) const { - auto tail_off = tail_offset(); - return idx >= tail_off - ? make_leaf_descent_pos(tail).visit(v, idx) - : visit_regular_descent(root, shift, v, idx); + auto tail_off = tail_offset(); + return idx >= tail_off ? make_leaf_descent_pos(tail).visit(v, idx) + : visit_regular_descent(root, shift, v, idx); } template @@ -241,18 +242,21 @@ struct rbtree template bool for_each_chunk_p(size_t first, size_t last, Fn&& fn) const { - return traverse_p(for_each_chunk_p_i_visitor{}, first, last, std::forward(fn)); + return traverse_p( + for_each_chunk_p_i_visitor{}, first, last, std::forward(fn)); } bool equals(const rbtree& other) const { - if (size != other.size) return false; - if (size == 0) return true; - return (size <= branches - || make_regular_sub_pos(root, shift, tail_offset()).visit( - equals_visitor{}, other.root)) - && make_leaf_sub_pos(tail, tail_size()).visit( - equals_visitor{}, other.tail); + if (size != other.size) + return false; + if (size == 0) + return true; + return (size <= branches || + make_regular_sub_pos(root, shift, tail_offset()) + .visit(equals_visitor{}, other.root)) && + make_leaf_sub_pos(tail, tail_size()) + .visit(equals_visitor{}, other.tail); } void ensure_mutable_tail(edit_t e, count_t n) @@ -267,29 +271,31 @@ struct rbtree void push_back_mut(edit_t e, T value) { auto tail_off = tail_offset(); - auto ts = size - tail_off; + auto ts = size - tail_off; if (ts < branches) { ensure_mutable_tail(e, ts); new (&tail->leaf()[ts]) T{std::move(value)}; } else { auto new_tail = node_t::make_leaf_e(e, std::move(value)); - try { + IMMER_TRY { if (tail_off == size_t{branches} << shift) { auto new_root = node_t::make_inner_e(e); - try { + IMMER_TRY { auto path = node_t::make_path_e(e, shift, tail); - new_root->inner() [0] = root; - new_root->inner() [1] = path; - root = new_root; - tail = new_tail; + new_root->inner()[0] = root; + new_root->inner()[1] = path; + root = new_root; + tail = new_tail; shift += B; - } catch (...) { + } + IMMER_CATCH (...) { node_t::delete_inner_e(new_root); - throw; + IMMER_RETHROW; } } else if (tail_off) { - auto new_root = make_regular_sub_pos(root, shift, tail_off) - .visit(push_tail_mut_visitor{}, e, tail); + auto new_root = + make_regular_sub_pos(root, shift, tail_off) + .visit(push_tail_mut_visitor{}, e, tail); root = new_root; tail = new_tail; } else { @@ -299,9 +305,10 @@ struct rbtree root = new_root; tail = new_tail; } - } catch (...) { + } + IMMER_CATCH (...) { node_t::delete_leaf(new_tail, 1); - throw; + IMMER_RETHROW; } } ++size; @@ -310,40 +317,43 @@ struct rbtree rbtree push_back(T value) const { auto tail_off = tail_offset(); - auto ts = size - tail_off; + auto ts = size - tail_off; if (ts < branches) { - auto new_tail = node_t::copy_leaf_emplace(tail, ts, - std::move(value)); - return { size + 1, shift, root->inc(), new_tail }; + auto new_tail = + node_t::copy_leaf_emplace(tail, ts, std::move(value)); + return {size + 1, shift, root->inc(), new_tail}; } else { auto new_tail = node_t::make_leaf_n(1, std::move(value)); - try { + IMMER_TRY { if (tail_off == size_t{branches} << shift) { auto new_root = node_t::make_inner_n(2); - try { - auto path = node_t::make_path(shift, tail); - new_root->inner() [0] = root; - new_root->inner() [1] = path; + IMMER_TRY { + auto path = node_t::make_path(shift, tail); + new_root->inner()[0] = root; + new_root->inner()[1] = path; root->inc(); tail->inc(); - return { size + 1, shift + B, new_root, new_tail }; - } catch (...) { + return {size + 1, shift + B, new_root, new_tail}; + } + IMMER_CATCH (...) { node_t::delete_inner(new_root, 2); - throw; + IMMER_RETHROW; } } else if (tail_off) { - auto new_root = make_regular_sub_pos(root, shift, tail_off) - .visit(push_tail_visitor{}, tail); + auto new_root = + make_regular_sub_pos(root, shift, tail_off) + .visit(push_tail_visitor{}, tail); tail->inc(); - return { size + 1, shift, new_root, new_tail }; + return {size + 1, shift, new_root, new_tail}; } else { auto new_root = node_t::make_path(shift, tail); tail->inc(); - return { size + 1, shift, new_root, new_tail }; + return {size + 1, shift, new_root, new_tail}; } - } catch (...) { + } + IMMER_CATCH (...) { node_t::delete_leaf(new_tail, 1); - throw; + IMMER_RETHROW; } } } @@ -358,7 +368,7 @@ struct rbtree auto tail_off = tail_offset(); if (idx >= tail_off) { ensure_mutable_tail(e, size - tail_off); - return tail->leaf() [idx & mask]; + return tail->leaf()[idx & mask]; } else { return make_regular_sub_pos(root, shift, tail_off) .visit(get_mut_visitor{}, idx, e, &root); @@ -373,67 +383,58 @@ struct rbtree const T& get_check(size_t index) const { if (index >= size) - throw std::out_of_range{"index out of range"}; + IMMER_THROW(std::out_of_range{"index out of range"}); return descend(get_visitor(), index); } - const T& front() const - { - return get(0); - } + const T& front() const { return get(0); } - const T& back() const - { - return tail->leaf()[(size - 1) & mask]; - } + const T& back() const { return tail->leaf()[(size - 1) & mask]; } template void update_mut(edit_t e, size_t idx, FnT&& fn) { auto& elem = get_mut(e, idx); - elem = std::forward(fn) (std::move(elem)); + elem = std::forward(fn)(std::move(elem)); } template rbtree update(size_t idx, FnT&& fn) const { - auto tail_off = tail_offset(); + auto tail_off = tail_offset(); if (idx >= tail_off) { auto tail_size = size - tail_off; - auto new_tail = make_leaf_sub_pos(tail, tail_size) - .visit(update_visitor{}, idx - tail_off, fn); - return { size, shift, root->inc(), new_tail }; + auto new_tail = + make_leaf_sub_pos(tail, tail_size) + .visit(update_visitor{}, idx - tail_off, fn); + return {size, shift, root->inc(), new_tail}; } else { - auto new_root = make_regular_sub_pos(root, shift, tail_off) - .visit(update_visitor{}, idx, fn); - return { size, shift, new_root, tail->inc() }; + auto new_root = make_regular_sub_pos(root, shift, tail_off) + .visit(update_visitor{}, idx, fn); + return {size, shift, new_root, tail->inc()}; } } void assoc_mut(edit_t e, size_t idx, T value) { - update_mut(e, idx, [&] (auto&&) { - return std::move(value); - }); + update_mut(e, idx, [&](auto&&) { return std::move(value); }); } rbtree assoc(size_t idx, T value) const { - return update(idx, [&] (auto&&) { - return std::move(value); - }); + return update(idx, [&](auto&&) { return std::move(value); }); } rbtree take(size_t new_size) const { auto tail_off = tail_offset(); if (new_size == 0) { - return empty(); + return {}; } else if (new_size >= size) { return *this; } else if (new_size > tail_off) { auto new_tail = node_t::copy_leaf(tail, new_size - tail_off); - return { new_size, shift, root->inc(), new_tail }; + return {new_size, shift, root->inc(), new_tail}; } else { using std::get; auto l = new_size - 1; @@ -443,11 +444,11 @@ struct rbtree auto new_root = get<1>(r); auto new_tail = get<3>(r); if (new_root) { - assert(new_root->compute_shift() == get<0>(r)); + IMMER_ASSERT_TAGGED(new_root->compute_shift() == get<0>(r)); assert(new_root->check(new_shift, new_size - get<2>(r))); - return { new_size, new_shift, new_root, new_tail }; + return {new_size, new_shift, new_root, new_tail}; } else { - return { new_size, BL, empty().root->inc(), new_tail }; + return {new_size, BL, empty_root(), new_tail}; } } } @@ -457,14 +458,14 @@ struct rbtree auto tail_off = tail_offset(); if (new_size == 0) { // todo: more efficient? - *this = empty(); + *this = {}; } else if (new_size >= size) { return; } else if (new_size > tail_off) { auto ts = size - tail_off; auto newts = new_size - tail_off; if (tail->can_mutate(e)) { - destroy_n(tail->leaf() + newts, ts - newts); + detail::destroy_n(tail->leaf() + newts, ts - newts); } else { auto new_tail = node_t::copy_leaf_e(e, tail, newts); dec_leaf(tail, ts); @@ -484,12 +485,12 @@ struct rbtree root = new_root; shift = new_shift; } else { - root = empty().root->inc(); + root = empty_root(); shift = BL; } dec_leaf(tail, size - tail_off); - size = new_size; - tail = new_tail; + size = new_size; + tail = new_tail; return; } } @@ -509,7 +510,7 @@ struct rbtree { #if IMMER_DEBUG_DEEP_CHECK if (tail_size() > 0) - assert(tail->check(0, tail_size())); + assert(tail->check(endshift, tail_size())); #endif return true; } @@ -520,7 +521,7 @@ struct rbtree if (tail_offset() > 0) assert(root->check(shift, tail_offset())); else { - assert(root->kind() == node_t::kind_t::inner); + IMMER_ASSERT_TAGGED(root->kind() == node_t::kind_t::inner); assert(shift == BL); } #endif diff --git a/src/immer/detail/rbts/rbtree_iterator.hpp b/src/immer/detail/rbts/rbtree_iterator.hpp index 672484eae3..90613b10b9 100644 --- a/src/immer/detail/rbts/rbtree_iterator.hpp +++ b/src/immer/detail/rbts/rbtree_iterator.hpp @@ -8,8 +8,8 @@ #pragma once -#include #include +#include namespace immer { namespace detail { @@ -26,22 +26,23 @@ struct rbtree_iterator { using tree_t = rbtree; - struct end_t {}; + struct end_t + {}; rbtree_iterator() = default; rbtree_iterator(const tree_t& v) - : v_ { &v } - , i_ { 0 } - , base_ { ~size_t{} } - , curr_ { nullptr } + : v_{&v} + , i_{0} + , base_{~size_t{}} + , curr_{nullptr} {} rbtree_iterator(const tree_t& v, end_t) - : v_ { &v } - , i_ { v.size } - , base_ { ~size_t{} } - , curr_ { nullptr } + : v_{&v} + , i_{v.size} + , base_{~size_t{}} + , curr_{nullptr} {} const tree_t& impl() const { return *v_; } @@ -50,9 +51,9 @@ struct rbtree_iterator private: friend iterator_core_access; - const tree_t* v_; - size_t i_; - mutable size_t base_; + const tree_t* v_; + size_t i_; + mutable size_t base_; mutable const T* curr_ = nullptr; void increment() @@ -74,16 +75,12 @@ struct rbtree_iterator i_ += n; } - bool equal(const rbtree_iterator& other) const - { - return i_ == other.i_; - } + bool equal(const rbtree_iterator& other) const { return i_ == other.i_; } std::ptrdiff_t distance_to(const rbtree_iterator& other) const { - return other.i_ > i_ - ? static_cast(other.i_ - i_) - : - static_cast(i_ - other.i_); + return other.i_ > i_ ? static_cast(other.i_ - i_) + : -static_cast(i_ - other.i_); } const T& dereference() const diff --git a/src/immer/detail/rbts/rrbtree.hpp b/src/immer/detail/rbts/rrbtree.hpp index 7d26e50a66..843921ba22 100644 --- a/src/immer/detail/rbts/rrbtree.hpp +++ b/src/immer/detail/rbts/rrbtree.hpp @@ -10,68 +10,80 @@ #include #include -#include #include +#include #include #include +#include #include #include +#include namespace immer { namespace detail { namespace rbts { -template +template struct rrbtree_iterator; -template +template struct rrbtree { using node_t = node; using edit_t = typename node_t::edit_t; using owner_t = typename MemoryPolicy::transience_t::owner; - size_t size; + size_t size; shift_t shift; node_t* root; node_t* tail; - static const rrbtree& empty() + constexpr static size_t max_size() + { + auto S = sizeof(size_t) * 8; + return ((size_t{1} << BL) - std::min(size_t{BL}, size_t{2})) * + ipow((size_t{1} << B) - 2, (S - BL) / B); + } + + static node_t* empty_root() + { + static const auto empty_ = []{ + constexpr auto size = node_t::sizeof_inner_n(0); + static std::aligned_storage_t storage; + return node_t::make_inner_n_into(&storage, size, 0u); + }(); + return empty_->inc(); + } + + static node_t* empty_tail() { - static const rrbtree empty_ { - 0, - BL, - node_t::make_inner_n(0u), - node_t::make_leaf_n(0u) - }; - return empty_; + static const auto empty_ = []{ + constexpr auto size = node_t::sizeof_leaf_n(0); + static std::aligned_storage_t storage; + return node_t::make_leaf_n_into(&storage, size, 0u); + }(); + return empty_->inc(); } template static auto from_initializer_list(std::initializer_list values) { - auto e = owner_t{}; - auto result = rrbtree{empty()}; + auto e = owner_t{}; + auto result = rrbtree{}; for (auto&& v : values) result.push_back_mut(e, v); return result; } - template , bool> = true> + template , bool> = true> static auto from_range(Iter first, Sent last) { - auto e = owner_t{}; - auto result = rrbtree{empty()}; + auto e = owner_t{}; + auto result = rrbtree{}; for (; first != last; ++first) result.push_back_mut(e, *first); return result; @@ -79,27 +91,39 @@ struct rrbtree static auto from_fill(size_t n, T v) { - auto e = owner_t{}; - auto result = rrbtree{empty()}; - while (n --> 0) + auto e = owner_t{}; + auto result = rrbtree{}; + while (n-- > 0) result.push_back_mut(e, v); return result; } - rrbtree(size_t sz, shift_t sh, node_t* r, node_t* t) - : size{sz}, shift{sh}, root{r}, tail{t} + rrbtree() noexcept + : size{0} + , shift{BL} + , root{empty_root()} + , tail{empty_tail()} { assert(check_tree()); } - rrbtree(const rrbtree& other) + rrbtree(size_t sz, shift_t sh, node_t* r, node_t* t) noexcept + : size{sz} + , shift{sh} + , root{r} + , tail{t} + { + assert(check_tree()); + } + + rrbtree(const rrbtree& other) noexcept : rrbtree{other.size, other.shift, other.root, other.tail} { inc(); } - rrbtree(rrbtree&& other) - : rrbtree{empty()} + rrbtree(rrbtree&& other) noexcept + : rrbtree{} { swap(*this, other); } @@ -111,25 +135,22 @@ struct rrbtree return *this; } - rrbtree& operator=(rrbtree&& other) + rrbtree& operator=(rrbtree&& other) noexcept { swap(*this, other); return *this; } - friend void swap(rrbtree& x, rrbtree& y) + friend void swap(rrbtree& x, rrbtree& y) noexcept { using std::swap; - swap(x.size, y.size); + swap(x.size, y.size); swap(x.shift, y.shift); - swap(x.root, y.root); - swap(x.tail, y.tail); + swap(x.root, y.root); + swap(x.tail, y.tail); } - ~rrbtree() - { - dec(); - } + ~rrbtree() { dec(); } void inc() const { @@ -137,24 +158,18 @@ struct rrbtree tail->inc(); } - void dec() const - { - traverse(dec_visitor()); - } + void dec() const { traverse(dec_visitor()); } - auto tail_size() const - { - return size - tail_offset(); - } + auto tail_size() const { return size - tail_offset(); } auto tail_offset() const { auto r = root->relaxed(); assert(r == nullptr || r->d.count); - return - r ? r->d.sizes[r->d.count - 1] : - size ? (size - 1) & ~mask - /* otherwise */ : 0; + return r ? r->d.sizes[r->d.count - 1] + : size ? (size - 1) & ~mask + /* otherwise */ + : 0; } template @@ -163,11 +178,15 @@ struct rrbtree auto tail_off = tail_offset(); auto tail_size = size - tail_off; - if (tail_off) visit_maybe_relaxed_sub(root, shift, tail_off, v, args...); - else make_empty_regular_pos(root).visit(v, args...); + if (tail_off) + visit_maybe_relaxed_sub(root, shift, tail_off, v, args...); + else + make_empty_regular_pos(root).visit(v, args...); - if (tail_size) make_leaf_sub_pos(tail, tail_size).visit(v, args...); - else make_empty_leaf_pos(tail).visit(v, args...); + if (tail_size) + make_leaf_sub_pos(tail, tail_size).visit(v, args...); + else + make_empty_leaf_pos(tail).visit(v, args...); } template @@ -177,16 +196,19 @@ struct rrbtree auto tail_size = size - tail_off; if (first < tail_off) - visit_maybe_relaxed_sub(root, shift, tail_off, v, + visit_maybe_relaxed_sub(root, + shift, + tail_off, + v, first, last < tail_off ? last : tail_off, args...); if (last > tail_off) - make_leaf_sub_pos(tail, tail_size).visit( - v, - first > tail_off ? first - tail_off : 0, - last - tail_off, - args...); + make_leaf_sub_pos(tail, tail_size) + .visit(v, + first > tail_off ? first - tail_off : 0, + last - tail_off, + args...); } template @@ -195,11 +217,10 @@ struct rrbtree auto tail_off = tail_offset(); auto tail_size = size - tail_off; return (tail_off - ? visit_maybe_relaxed_sub(root, shift, tail_off, v, args...) - : make_empty_regular_pos(root).visit(v, args...)) - && (tail_size - ? make_leaf_sub_pos(tail, tail_size).visit(v, args...) - : make_empty_leaf_pos(tail).visit(v, args...)); + ? visit_maybe_relaxed_sub(root, shift, tail_off, v, args...) + : make_empty_regular_pos(root).visit(v, args...)) && + (tail_size ? make_leaf_sub_pos(tail, tail_size).visit(v, args...) + : make_empty_leaf_pos(tail).visit(v, args...)); } template @@ -207,29 +228,31 @@ struct rrbtree { auto tail_off = tail_offset(); auto tail_size = size - tail_off; - return - (first < tail_off - ? visit_maybe_relaxed_sub(root, shift, tail_off, v, - first, - last < tail_off ? last : tail_off, - args...) - : true) - && (last > tail_off - ? make_leaf_sub_pos(tail, tail_size).visit( - v, - first > tail_off ? first - tail_off : 0, - last - tail_off, - args...) - : true); + return (first < tail_off + ? visit_maybe_relaxed_sub(root, + shift, + tail_off, + v, + first, + last < tail_off ? last : tail_off, + args...) + : true) && + (last > tail_off + ? make_leaf_sub_pos(tail, tail_size) + .visit(v, + first > tail_off ? first - tail_off : 0, + last - tail_off, + args...) + : true); } template decltype(auto) descend(Visitor v, size_t idx) const { - auto tail_off = tail_offset(); + auto tail_off = tail_offset(); return idx >= tail_off - ? make_leaf_descent_pos(tail).visit(v, idx - tail_off) - : visit_maybe_relaxed_descent(root, shift, v, idx); + ? make_leaf_descent_pos(tail).visit(v, idx - tail_off) + : visit_maybe_relaxed_descent(root, shift, v, idx); } template @@ -253,15 +276,18 @@ struct rrbtree template bool for_each_chunk_p(size_t first, size_t last, Fn&& fn) const { - return traverse_p(for_each_chunk_p_i_visitor{}, first, last, std::forward(fn)); + return traverse_p( + for_each_chunk_p_i_visitor{}, first, last, std::forward(fn)); } bool equals(const rrbtree& other) const { using iter_t = rrbtree_iterator; - if (size != other.size) return false; - if (size == 0) return true; - auto tail_off = tail_offset(); + if (size != other.size) + return false; + if (size == 0) + return true; + auto tail_off = tail_offset(); auto tail_off_other = other.tail_offset(); // compare trees if (tail_off > 0 && tail_off_other > 0) { @@ -269,114 +295,137 @@ struct rrbtree // relaxed trees that sadly we haven't managed to exercise // in tests yet... if (other.shift >= shift) { - if (!visit_maybe_relaxed_sub( - other.root, other.shift, tail_off_other, - equals_visitor::rrb{}, iter_t{other}, - root, shift, tail_off)) + if (!visit_maybe_relaxed_sub(other.root, + other.shift, + tail_off_other, + equals_visitor::rrb{}, + iter_t{other}, + root, + shift, + tail_off)) return false; } else { - if (!visit_maybe_relaxed_sub( - root, shift, tail_off, - equals_visitor::rrb{}, iter_t{*this}, - other.root, other.shift, tail_off_other)) + if (!visit_maybe_relaxed_sub(root, + shift, + tail_off, + equals_visitor::rrb{}, + iter_t{*this}, + other.root, + other.shift, + tail_off_other)) return false; } } - return - tail_off == tail_off_other ? make_leaf_sub_pos( - tail, tail_size()).visit( - equals_visitor{}, other.tail) : - tail_off > tail_off_other ? std::equal( - tail->leaf(), tail->leaf() + (size - tail_off), - other.tail->leaf() + (tail_off - tail_off_other)) - /* otherwise */ : std::equal( - tail->leaf(), tail->leaf() + (size - tail_off), - iter_t{other} + tail_off); + return tail_off == tail_off_other + ? make_leaf_sub_pos(tail, tail_size()) + .visit(equals_visitor{}, other.tail) + : tail_off > tail_off_other + ? std::equal(tail->leaf(), + tail->leaf() + (size - tail_off), + other.tail->leaf() + + (tail_off - tail_off_other)) + /* otherwise */ + : std::equal(tail->leaf(), + tail->leaf() + (size - tail_off), + iter_t{other} + tail_off); } - std::tuple - push_tail(node_t* root, shift_t shift, size_t size, - node_t* tail, count_t tail_size) const + std::tuple push_tail(node_t* root, + shift_t shift, + size_t size, + node_t* tail, + count_t tail_size) const { if (auto r = root->relaxed()) { - auto new_root = make_relaxed_pos(root, shift, r) - .visit(push_tail_visitor{}, tail, tail_size); + auto new_root = + make_relaxed_pos(root, shift, r) + .visit(push_tail_visitor{}, tail, tail_size); if (new_root) - return { shift, new_root }; + return std::make_tuple(shift, new_root); else { auto new_root = node_t::make_inner_r_n(2); - try { - auto new_path = node_t::make_path(shift, tail); - new_root->inner() [0] = root->inc(); - new_root->inner() [1] = new_path; - new_root->relaxed()->d.sizes [0] = size; - new_root->relaxed()->d.sizes [1] = size + tail_size; + IMMER_TRY { + auto new_path = node_t::make_path(shift, tail); + new_root->inner()[0] = root->inc(); + new_root->inner()[1] = new_path; + new_root->relaxed()->d.sizes[0] = size; + new_root->relaxed()->d.sizes[1] = size + tail_size; + assert(size); + assert(tail_size); new_root->relaxed()->d.count = 2u; - } catch (...) { + } + IMMER_CATCH (...) { node_t::delete_inner_r(new_root, 2); - throw; + IMMER_RETHROW; } - return { shift + B, new_root }; + return std::make_tuple(shift + B, new_root); } } else if (size == size_t{branches} << shift) { auto new_root = node_t::make_inner_n(2); - try { - auto new_path = node_t::make_path(shift, tail); - new_root->inner() [0] = root->inc(); - new_root->inner() [1] = new_path; - } catch (...) { + IMMER_TRY { + auto new_path = node_t::make_path(shift, tail); + new_root->inner()[0] = root->inc(); + new_root->inner()[1] = new_path; + } + IMMER_CATCH (...) { node_t::delete_inner(new_root, 2); - throw; + IMMER_RETHROW; } - return { shift + B, new_root }; + return std::make_tuple(shift + B, new_root); } else if (size) { auto new_root = make_regular_sub_pos(root, shift, size) - .visit(push_tail_visitor{}, tail); - return { shift, new_root }; + .visit(push_tail_visitor{}, tail); + return std::make_tuple(shift, new_root); } else { - return { shift, node_t::make_path(shift, tail) }; + return std::make_tuple(shift, node_t::make_path(shift, tail)); } } - void push_tail_mut(edit_t e, size_t tail_off, - node_t* tail, count_t tail_size) + void + push_tail_mut(edit_t e, size_t tail_off, node_t* tail, count_t tail_size) { if (auto r = root->relaxed()) { - auto new_root = make_relaxed_pos(root, shift, r) - .visit(push_tail_mut_visitor{}, e, tail, tail_size); + auto new_root = + make_relaxed_pos(root, shift, r) + .visit(push_tail_mut_visitor{}, e, tail, tail_size); if (new_root) { root = new_root; } else { auto new_root = node_t::make_inner_r_e(e); - try { - auto new_path = node_t::make_path_e(e, shift, tail); - new_root->inner() [0] = root; - new_root->inner() [1] = new_path; - new_root->relaxed()->d.sizes [0] = tail_off; - new_root->relaxed()->d.sizes [1] = tail_off + tail_size; + IMMER_TRY { + auto new_path = node_t::make_path_e(e, shift, tail); + new_root->inner()[0] = root; + new_root->inner()[1] = new_path; + new_root->relaxed()->d.sizes[0] = tail_off; + new_root->relaxed()->d.sizes[1] = tail_off + tail_size; + assert(tail_off); + assert(tail_size); new_root->relaxed()->d.count = 2u; - root = new_root; + root = new_root; shift += B; - } catch (...) { + } + IMMER_CATCH (...) { node_t::delete_inner_r_e(new_root); - throw; + IMMER_RETHROW; } } } else if (tail_off == size_t{branches} << shift) { auto new_root = node_t::make_inner_e(e); - try { - auto new_path = node_t::make_path_e(e, shift, tail); - new_root->inner() [0] = root; - new_root->inner() [1] = new_path; - root = new_root; + IMMER_TRY { + auto new_path = node_t::make_path_e(e, shift, tail); + new_root->inner()[0] = root; + new_root->inner()[1] = new_path; + root = new_root; shift += B; - } catch (...) { + } + IMMER_CATCH (...) { node_t::delete_inner_e(new_root); - throw; + IMMER_RETHROW; } } else if (tail_off) { - auto new_root = make_regular_sub_pos(root, shift, tail_off) - .visit(push_tail_mut_visitor{}, e, tail); + auto new_root = + make_regular_sub_pos(root, shift, tail_off) + .visit(push_tail_mut_visitor{}, e, tail); root = new_root; } else { auto new_root = node_t::make_path_e(e, shift, tail); @@ -404,12 +453,13 @@ struct rrbtree using std::get; auto new_tail = node_t::make_leaf_e(e, std::move(value)); auto tail_off = tail_offset(); - try { + IMMER_TRY { push_tail_mut(e, tail_off, tail, ts); tail = new_tail; - } catch (...) { + } + IMMER_CATCH (...) { node_t::delete_leaf(new_tail, 1u); - throw; + IMMER_RETHROW; } } ++size; @@ -419,39 +469,38 @@ struct rrbtree { auto ts = tail_size(); if (ts < branches) { - auto new_tail = node_t::copy_leaf_emplace(tail, ts, - std::move(value)); - return { size + 1, shift, root->inc(), new_tail }; + auto new_tail = + node_t::copy_leaf_emplace(tail, ts, std::move(value)); + return {size + 1, shift, root->inc(), new_tail}; } else { using std::get; auto new_tail = node_t::make_leaf_n(1u, std::move(value)); auto tail_off = tail_offset(); - try { - auto new_root = push_tail(root, shift, tail_off, - tail, size - tail_off); + IMMER_TRY { + auto new_root = + push_tail(root, shift, tail_off, tail, size - tail_off); tail->inc(); - return { size + 1, get<0>(new_root), get<1>(new_root), new_tail }; - } catch (...) { + return {size + 1, get<0>(new_root), get<1>(new_root), new_tail}; + } + IMMER_CATCH (...) { node_t::delete_leaf(new_tail, 1u); - throw; + IMMER_RETHROW; } } } - std::tuple - region_for(size_t idx) const + std::tuple region_for(size_t idx) const { using std::get; auto tail_off = tail_offset(); if (idx >= tail_off) { - return { tail->leaf(), tail_off, size }; + return std::make_tuple(tail->leaf(), tail_off, size); } else { auto subs = visit_maybe_relaxed_sub( - root, shift, tail_off, - region_for_visitor(), idx); + root, shift, tail_off, region_for_visitor(), idx); auto first = idx - get<1>(subs); auto end = first + get<2>(subs); - return { get<0>(subs), first, end }; + return std::make_tuple(get<0>(subs), first, end); } } @@ -460,11 +509,15 @@ struct rrbtree auto tail_off = tail_offset(); if (idx >= tail_off) { ensure_mutable_tail(e, size - tail_off); - return tail->leaf() [(idx - tail_off) & mask]; + return tail->leaf()[(idx - tail_off) & mask]; } else { - return visit_maybe_relaxed_sub( - root, shift, tail_off, - get_mut_visitor{}, idx, e, &root); + return visit_maybe_relaxed_sub(root, + shift, + tail_off, + get_mut_visitor{}, + idx, + e, + &root); } } @@ -476,70 +529,60 @@ struct rrbtree const T& get_check(size_t index) const { if (index >= size) - throw std::out_of_range{"out of range"}; + IMMER_THROW(std::out_of_range{"out of range"}); return descend(get_visitor(), index); } - const T& front() const - { - return get(0); - } + const T& front() const { return get(0); } - const T& back() const - { - return get(size - 1); - } + const T& back() const { return get(size - 1); } template void update_mut(edit_t e, size_t idx, FnT&& fn) { auto& elem = get_mut(e, idx); - elem = std::forward(fn) (std::move(elem)); + elem = std::forward(fn)(std::move(elem)); } template rrbtree update(size_t idx, FnT&& fn) const { - auto tail_off = tail_offset(); + auto tail_off = tail_offset(); if (idx >= tail_off) { auto tail_size = size - tail_off; - auto new_tail = make_leaf_sub_pos(tail, tail_size) - .visit(update_visitor{}, idx - tail_off, fn); - return { size, shift, root->inc(), new_tail }; + auto new_tail = + make_leaf_sub_pos(tail, tail_size) + .visit(update_visitor{}, idx - tail_off, fn); + return {size, shift, root->inc(), new_tail}; } else { - auto new_root = visit_maybe_relaxed_sub( - root, shift, tail_off, - update_visitor{}, idx, fn); - return { size, shift, new_root, tail->inc() }; + auto new_root = visit_maybe_relaxed_sub( + root, shift, tail_off, update_visitor{}, idx, fn); + return {size, shift, new_root, tail->inc()}; } } void assoc_mut(edit_t e, size_t idx, T value) { - update_mut(e, idx, [&] (auto&&) { - return std::move(value); - }); + update_mut(e, idx, [&](auto&&) { return std::move(value); }); } rrbtree assoc(size_t idx, T value) const { - return update(idx, [&] (auto&&) { - return std::move(value); - }); + return update(idx, [&](auto&&) { return std::move(value); }); } void take_mut(edit_t e, size_t new_size) { auto tail_off = tail_offset(); if (new_size == 0) { - *this = empty(); + *this = {}; } else if (new_size >= size) { return; } else if (new_size > tail_off) { auto ts = size - tail_off; auto newts = new_size - tail_off; if (tail->can_mutate(e)) { - destroy_n(tail->leaf() + newts, ts - newts); + detail::destroy_n(tail->leaf() + newts, ts - newts); } else { auto new_tail = node_t::copy_leaf_e(e, tail, newts); dec_leaf(tail, ts); @@ -559,12 +602,12 @@ struct rrbtree root = new_root; shift = new_shift; } else { - root = empty().root->inc(); + root = empty_root(); shift = BL; } dec_leaf(tail, size - tail_off); - size = new_size; - tail = new_tail; + size = new_size; + tail = new_tail; return; } } @@ -573,12 +616,12 @@ struct rrbtree { auto tail_off = tail_offset(); if (new_size == 0) { - return empty(); + return {}; } else if (new_size >= size) { return *this; } else if (new_size > tail_off) { auto new_tail = node_t::copy_leaf(tail, new_size - tail_off); - return { new_size, shift, root->inc(), new_tail }; + return {new_size, shift, root->inc(), new_tail}; } else { using std::get; auto l = new_size - 1; @@ -588,11 +631,11 @@ struct rrbtree auto new_root = get<1>(r); auto new_tail = get<3>(r); if (new_root) { - assert(new_root->compute_shift() == get<0>(r)); + IMMER_ASSERT_TAGGED(new_root->compute_shift() == get<0>(r)); assert(new_root->check(new_shift, new_size - get<2>(r))); - return { new_size, new_shift, new_root, new_tail }; + return {new_size, new_shift, new_root, new_tail}; } else { - return { new_size, BL, empty().root->inc(), new_tail }; + return {new_size, BL, empty_root(), new_tail}; } } } @@ -604,27 +647,28 @@ struct rrbtree if (elems == 0) { return; } else if (elems >= size) { - *this = empty(); + *this = {}; } else if (elems == tail_off) { dec_inner(root, shift, tail_off); shift = BL; - root = empty().root->inc(); + root = empty_root(); size -= elems; return; } else if (elems > tail_off) { auto v = slice_left_mut_visitor(); - tail = get<1>(make_leaf_sub_pos(tail, size - tail_off).visit( - v, elems - tail_off, e)); - if (root != empty().root) { + tail = get<1>(make_leaf_sub_pos(tail, size - tail_off) + .visit(v, elems - tail_off, e)); + if (tail_off) { dec_inner(root, shift, tail_off); shift = BL; - root = empty().root->inc(); + root = empty_root(); } size -= elems; return; } else { auto v = slice_left_mut_visitor(); - auto r = visit_maybe_relaxed_sub(root, shift, tail_off, v, elems, e); + auto r = + visit_maybe_relaxed_sub(root, shift, tail_off, v, elems, e); shift = get<0>(r); root = get<1>(r); size -= elems; @@ -637,28 +681,28 @@ struct rrbtree if (elems == 0) { return *this; } else if (elems >= size) { - return empty(); + return {}; } else if (elems == tail_offset()) { - return { size - elems, BL, empty().root->inc(), tail->inc() }; + return {size - elems, BL, empty_root(), tail->inc()}; } else if (elems > tail_offset()) { auto tail_off = tail_offset(); - auto new_tail = node_t::copy_leaf(tail, elems - tail_off, - size - tail_off); - return { size - elems, BL, empty().root->inc(), new_tail }; + auto new_tail = + node_t::copy_leaf(tail, elems - tail_off, size - tail_off); + return {size - elems, BL, empty_root(), new_tail}; } else { using std::get; auto v = slice_left_visitor(); - auto r = visit_maybe_relaxed_sub(root, shift, tail_offset(), v, elems); + auto r = + visit_maybe_relaxed_sub(root, shift, tail_offset(), v, elems); auto new_root = get<1>(r); auto new_shift = get<0>(r); - return { size - elems, new_shift, new_root, tail->inc() }; + return {size - elems, new_shift, new_root, tail->inc()}; } - return *this; } rrbtree concat(const rrbtree& r) const { - assert(r.size < (std::numeric_limits::max() - size)); + assert(r.size + size <= max_size()); using std::get; if (size == 0) return r; @@ -669,57 +713,68 @@ struct rrbtree auto tail_offst = tail_offset(); auto tail_size = size - tail_offst; if (tail_size == branches) { - auto new_root = push_tail(root, shift, tail_offst, - tail, tail_size); + auto new_root = + push_tail(root, shift, tail_offst, tail, tail_size); tail->inc(); - return { size + r.size, get<0>(new_root), get<1>(new_root), - r.tail->inc() }; + return {size + r.size, + get<0>(new_root), + get<1>(new_root), + r.tail->inc()}; } else if (tail_size + r.size <= branches) { - auto new_tail = node_t::copy_leaf(tail, tail_size, - r.tail, r.size); - return { size + r.size, shift, root->inc(), new_tail }; + auto new_tail = + node_t::copy_leaf(tail, tail_size, r.tail, r.size); + return {size + r.size, shift, root->inc(), new_tail}; } else { auto remaining = branches - tail_size; - auto add_tail = node_t::copy_leaf(tail, tail_size, - r.tail, remaining); - try { - auto new_tail = node_t::copy_leaf(r.tail, remaining, r.size); - try { - auto new_root = push_tail(root, shift, tail_offst, - add_tail, branches); - return { size + r.size, - get<0>(new_root), get<1>(new_root), - new_tail }; - } catch (...) { + auto add_tail = + node_t::copy_leaf(tail, tail_size, r.tail, remaining); + IMMER_TRY { + auto new_tail = + node_t::copy_leaf(r.tail, remaining, r.size); + IMMER_TRY { + auto new_root = push_tail( + root, shift, tail_offst, add_tail, branches); + return {size + r.size, + get<0>(new_root), + get<1>(new_root), + new_tail}; + } + IMMER_CATCH (...) { node_t::delete_leaf(new_tail, r.size - remaining); - throw; + IMMER_RETHROW; } - } catch (...) { + } + IMMER_CATCH (...) { node_t::delete_leaf(add_tail, branches); - throw; + IMMER_RETHROW; } } } else if (tail_offset() == 0) { auto tail_offst = tail_offset(); auto tail_size = size - tail_offst; - auto concated = concat_trees(tail, tail_size, - r.root, r.shift, r.tail_offset()); - auto new_shift = concated.shift(); - auto new_root = concated.node(); - assert(new_shift == new_root->compute_shift()); + auto concated = + concat_trees(tail, tail_size, r.root, r.shift, r.tail_offset()); + auto new_shift = concated.shift(); + auto new_root = concated.node(); + IMMER_ASSERT_TAGGED(new_shift == new_root->compute_shift()); assert(new_root->check(new_shift, size + r.tail_offset())); - return { size + r.size, new_shift, new_root, r.tail->inc() }; + return {size + r.size, new_shift, new_root, r.tail->inc()}; } else { auto tail_offst = tail_offset(); auto tail_size = size - tail_offst; - auto concated = concat_trees(root, shift, tail_offst, - tail, tail_size, - r.root, r.shift, r.tail_offset()); + auto concated = concat_trees(root, + shift, + tail_offst, + tail, + tail_size, + r.root, + r.shift, + r.tail_offset()); auto new_shift = concated.shift(); auto new_root = concated.node(); - assert(new_shift == new_root->compute_shift()); + IMMER_ASSERT_TAGGED(new_shift == new_root->compute_shift()); assert(new_root->check(new_shift, size + r.tail_offset())); - return { size + r.size, new_shift, new_root, r.tail->inc() }; + return {size + r.size, new_shift, new_root, r.tail->inc()}; } } @@ -746,80 +801,108 @@ struct rrbtree return; } else if (tail_size + r.size <= branches) { l.ensure_mutable_tail(el, tail_size); - std::uninitialized_copy(r.tail->leaf(), - r.tail->leaf() + r.size, - l.tail->leaf() + tail_size); + detail::uninitialized_copy(r.tail->leaf(), + r.tail->leaf() + r.size, + l.tail->leaf() + tail_size); l.size += r.size; return; } else { auto remaining = branches - tail_size; l.ensure_mutable_tail(el, tail_size); - std::uninitialized_copy(r.tail->leaf(), - r.tail->leaf() + remaining, - l.tail->leaf() + tail_size); - try { - auto new_tail = node_t::copy_leaf_e(el, r.tail, remaining, r.size); - try { + detail::uninitialized_copy(r.tail->leaf(), + r.tail->leaf() + remaining, + l.tail->leaf() + tail_size); + IMMER_TRY { + auto new_tail = + node_t::copy_leaf_e(el, r.tail, remaining, r.size); + IMMER_TRY { l.push_tail_mut(el, tail_offst, l.tail, branches); l.tail = new_tail; l.size += r.size; return; - } catch (...) { + } + IMMER_CATCH (...) { node_t::delete_leaf(new_tail, r.size - remaining); - throw; + IMMER_RETHROW; } - } catch (...) { - destroy_n(r.tail->leaf() + tail_size, remaining); - throw; + } + IMMER_CATCH (...) { + detail::destroy_n(r.tail->leaf() + tail_size, remaining); + IMMER_RETHROW; } } } else if (l.tail_offset() == 0) { if (supports_transient_concat) { auto tail_offst = l.tail_offset(); auto tail_size = l.size - tail_offst; - auto concated = concat_trees_mut( - el, - el, l.tail, tail_size, - MemoryPolicy::transience_t::noone, - r.root, r.shift, r.tail_offset()); - assert(concated.shift() == concated.node()->compute_shift()); - assert(concated.node()->check(concated.shift(), l.size + r.tail_offset())); + auto concated = + concat_trees_mut(el, + el, + l.tail, + tail_size, + MemoryPolicy::transience_t::noone, + r.root, + r.shift, + r.tail_offset()); + IMMER_ASSERT_TAGGED(concated.shift() == + concated.node()->compute_shift()); l.size += r.size; l.shift = concated.shift(); l.root = concated.node(); l.tail = r.tail; + assert(l.check_tree()); } else { auto tail_offst = l.tail_offset(); auto tail_size = l.size - tail_offst; - auto concated = concat_trees(l.tail, tail_size, - r.root, r.shift, r.tail_offset()); - l = { l.size + r.size, concated.shift(), - concated.node(), r.tail->inc() }; + auto concated = concat_trees( + l.tail, tail_size, r.root, r.shift, r.tail_offset()); + l = {l.size + r.size, + concated.shift(), + concated.node(), + r.tail->inc()}; + assert(l.check_tree()); return; } } else { if (supports_transient_concat) { auto tail_offst = l.tail_offset(); auto tail_size = l.size - tail_offst; - auto concated = concat_trees_mut( - el, - el, l.root, l.shift, tail_offst, l.tail, tail_size, - MemoryPolicy::transience_t::noone, - r.root, r.shift, r.tail_offset()); - assert(concated.shift() == concated.node()->compute_shift()); - assert(concated.node()->check(concated.shift(), l.size + r.tail_offset())); + assert(l.check_tree()); + assert(r.check_tree()); + auto concated = + concat_trees_mut(el, + el, + l.root, + l.shift, + tail_offst, + l.tail, + tail_size, + MemoryPolicy::transience_t::noone, + r.root, + r.shift, + r.tail_offset()); + IMMER_ASSERT_TAGGED(concated.shift() == + concated.node()->compute_shift()); l.size += r.size; l.shift = concated.shift(); l.root = concated.node(); l.tail = r.tail; + assert(l.check_tree()); } else { auto tail_offst = l.tail_offset(); auto tail_size = l.size - tail_offst; - auto concated = concat_trees(l.root, l.shift, tail_offst, - l.tail, tail_size, - r.root, r.shift, r.tail_offset()); - l = { l.size + r.size, concated.shift(), - concated.node(), r.tail->inc() }; + auto concated = concat_trees(l.root, + l.shift, + tail_offst, + l.tail, + tail_size, + r.root, + r.shift, + r.tail_offset()); + l = {l.size + r.size, + concated.shift(), + concated.node(), + r.tail->inc()}; } } } @@ -841,13 +924,12 @@ struct rrbtree // this could be improved by making sure that the // newly created nodes as part of the `push_tail()` // are tagged with `er` - auto res = l.push_tail(l.root, l.shift, tail_offst, - l.tail, tail_size); + auto res = + l.push_tail(l.root, l.shift, tail_offst, l.tail, tail_size); l.tail->inc(); // note: leak if mutably concatenated // with itself, but this is forbidden // by the interface - r = { l.size + r.size, get<0>(res), get<1>(res), - r.tail->inc() }; + r = {l.size + r.size, get<0>(res), get<1>(res), r.tail->inc()}; return; } else if (tail_size + r.size <= branches) { // doing this in a exception way mutating way is very @@ -857,83 +939,111 @@ struct rrbtree // // we could however improve this by at least moving the // elements of the right tail... - auto new_tail = node_t::copy_leaf(l.tail, tail_size, - r.tail, r.size); - r = { l.size + r.size, l.shift, l.root->inc(), new_tail }; + auto new_tail = + node_t::copy_leaf(l.tail, tail_size, r.tail, r.size); + r = {l.size + r.size, l.shift, l.root->inc(), new_tail}; return; } else { // like the immutable version auto remaining = branches - tail_size; - auto add_tail = node_t::copy_leaf_e(er, - l.tail, tail_size, - r.tail, remaining); - try { - auto new_tail = node_t::copy_leaf_e(er, r.tail, remaining, r.size); - try { + auto add_tail = node_t::copy_leaf_e( + er, l.tail, tail_size, r.tail, remaining); + IMMER_TRY { + auto new_tail = + node_t::copy_leaf_e(er, r.tail, remaining, r.size); + IMMER_TRY { // this could be improved by making sure that the // newly created nodes as part of the `push_tail()` // are tagged with `er` - auto new_root = l.push_tail(l.root, l.shift, tail_offst, - add_tail, branches); - r = { l.size + r.size, - get<0>(new_root), get<1>(new_root), - new_tail }; + auto new_root = l.push_tail(l.root, + l.shift, + tail_offst, + add_tail, + branches); + r = {l.size + r.size, + get<0>(new_root), + get<1>(new_root), + new_tail}; return; - } catch (...) { + } + IMMER_CATCH (...) { node_t::delete_leaf(new_tail, r.size - remaining); - throw; + IMMER_RETHROW; } - } catch (...) { + } + IMMER_CATCH (...) { node_t::delete_leaf(add_tail, branches); - throw; + IMMER_RETHROW; } - return; } } else if (l.tail_offset() == 0) { if (supports_transient_concat) { auto tail_offst = l.tail_offset(); auto tail_size = l.size - tail_offst; - auto concated = concat_trees_mut( - er, - MemoryPolicy::transience_t::noone, l.tail, tail_size, - er,r.root, r.shift, r.tail_offset()); - assert(concated.shift() == concated.node()->compute_shift()); - assert(concated.node()->check(concated.shift(), l.size + r.tail_offset())); + auto concated = + concat_trees_mut(er, + MemoryPolicy::transience_t::noone, + l.tail, + tail_size, + er, + r.root, + r.shift, + r.tail_offset()); + IMMER_ASSERT_TAGGED(concated.shift() == + concated.node()->compute_shift()); r.size += l.size; r.shift = concated.shift(); r.root = concated.node(); + assert(r.check_tree()); } else { auto tail_offst = l.tail_offset(); auto tail_size = l.size - tail_offst; - auto concated = concat_trees(l.tail, tail_size, - r.root, r.shift, r.tail_offset()); - r = { l.size + r.size, concated.shift(), - concated.node(), r.tail->inc() }; + auto concated = concat_trees( + l.tail, tail_size, r.root, r.shift, r.tail_offset()); + r = {l.size + r.size, + concated.shift(), + concated.node(), + r.tail->inc()}; return; } } else { if (supports_transient_concat) { auto tail_offst = l.tail_offset(); auto tail_size = l.size - tail_offst; - auto concated = concat_trees_mut( - er, - MemoryPolicy::transience_t::noone, - l.root, l.shift, tail_offst, l.tail, tail_size, - er, r.root, r.shift, r.tail_offset()); - assert(concated.shift() == concated.node()->compute_shift()); - assert(concated.node()->check(concated.shift(), l.size + r.tail_offset())); + auto concated = + concat_trees_mut(er, + MemoryPolicy::transience_t::noone, + l.root, + l.shift, + tail_offst, + l.tail, + tail_size, + er, + r.root, + r.shift, + r.tail_offset()); + IMMER_ASSERT_TAGGED(concated.shift() == + concated.node()->compute_shift()); r.size += l.size; r.shift = concated.shift(); r.root = concated.node(); + assert(r.check_tree()); return; } else { auto tail_offst = l.tail_offset(); auto tail_size = l.size - tail_offst; - auto concated = concat_trees(l.root, l.shift, tail_offst, - l.tail, tail_size, - r.root, r.shift, r.tail_offset()); - r = { l.size + r.size, concated.shift(), - concated.node(), r.tail->inc() }; + auto concated = concat_trees(l.root, + l.shift, + tail_offst, + l.tail, + tail_size, + r.root, + r.shift, + r.tail_offset()); + r = {l.size + r.size, + concated.shift(), + concated.node(), + r.tail->inc()}; return; } } @@ -961,12 +1071,12 @@ struct rrbtree l.ensure_mutable_tail(el, tail_size); if (r.tail->can_mutate(er)) detail::uninitialized_move(r.tail->leaf(), - r.tail->leaf() + r.size, - l.tail->leaf() + tail_size); + r.tail->leaf() + r.size, + l.tail->leaf() + tail_size); else - std::uninitialized_copy(r.tail->leaf(), - r.tail->leaf() + r.size, - l.tail->leaf() + tail_size); + detail::uninitialized_copy(r.tail->leaf(), + r.tail->leaf() + r.size, + l.tail->leaf() + tail_size); l.size += r.size; return; } else { @@ -974,75 +1084,103 @@ struct rrbtree l.ensure_mutable_tail(el, tail_size); if (r.tail->can_mutate(er)) detail::uninitialized_move(r.tail->leaf(), - r.tail->leaf() + remaining, - l.tail->leaf() + tail_size); + r.tail->leaf() + remaining, + l.tail->leaf() + tail_size); else - std::uninitialized_copy(r.tail->leaf(), - r.tail->leaf() + remaining, - l.tail->leaf() + tail_size); - try { - auto new_tail = node_t::copy_leaf_e(el, r.tail, remaining, r.size); - try { + detail::uninitialized_copy(r.tail->leaf(), + r.tail->leaf() + remaining, + l.tail->leaf() + tail_size); + IMMER_TRY { + auto new_tail = + node_t::copy_leaf_e(el, r.tail, remaining, r.size); + IMMER_TRY { l.push_tail_mut(el, tail_offst, l.tail, branches); l.tail = new_tail; l.size += r.size; return; - } catch (...) { + } + IMMER_CATCH (...) { node_t::delete_leaf(new_tail, r.size - remaining); - throw; + IMMER_RETHROW; } - } catch (...) { - destroy_n(r.tail->leaf() + tail_size, remaining); - throw; + } + IMMER_CATCH (...) { + detail::destroy_n(r.tail->leaf() + tail_size, remaining); + IMMER_RETHROW; } } } else if (l.tail_offset() == 0) { if (supports_transient_concat) { auto tail_offst = l.tail_offset(); auto tail_size = l.size - tail_offst; - auto concated = concat_trees_mut( - el, - el, l.tail, tail_size, - er, r.root, r.shift, r.tail_offset()); - assert(concated.shift() == concated.node()->compute_shift()); - assert(concated.node()->check(concated.shift(), l.size + r.tail_offset())); + auto concated = concat_trees_mut(el, + el, + l.tail, + tail_size, + er, + r.root, + r.shift, + r.tail_offset()); + IMMER_ASSERT_TAGGED(concated.shift() == + concated.node()->compute_shift()); l.size += r.size; l.shift = concated.shift(); l.root = concated.node(); l.tail = r.tail; + assert(l.check_tree()); r.hard_reset(); + return; } else { auto tail_offst = l.tail_offset(); auto tail_size = l.size - tail_offst; - auto concated = concat_trees(l.tail, tail_size, - r.root, r.shift, r.tail_offset()); - l = { l.size + r.size, concated.shift(), - concated.node(), r.tail->inc() }; + auto concated = concat_trees( + l.tail, tail_size, r.root, r.shift, r.tail_offset()); + l = {l.size + r.size, + concated.shift(), + concated.node(), + r.tail->inc()}; return; } } else { if (supports_transient_concat) { auto tail_offst = l.tail_offset(); auto tail_size = l.size - tail_offst; - auto concated = concat_trees_mut( - el, - el, l.root, l.shift, tail_offst, l.tail, tail_size, - er, r.root, r.shift, r.tail_offset()); - assert(concated.shift() == concated.node()->compute_shift()); - assert(concated.node()->check(concated.shift(), l.size + r.tail_offset())); + auto concated = concat_trees_mut(el, + el, + l.root, + l.shift, + tail_offst, + l.tail, + tail_size, + er, + r.root, + r.shift, + r.tail_offset()); + IMMER_ASSERT_TAGGED(concated.shift() == + concated.node()->compute_shift()); l.size += r.size; l.shift = concated.shift(); l.root = concated.node(); l.tail = r.tail; + assert(l.check_tree()); r.hard_reset(); + return; } else { auto tail_offst = l.tail_offset(); auto tail_size = l.size - tail_offst; - auto concated = concat_trees(l.root, l.shift, tail_offst, - l.tail, tail_size, - r.root, r.shift, r.tail_offset()); - l = { l.size + r.size, concated.shift(), - concated.node(), r.tail->inc() }; + auto concated = concat_trees(l.root, + l.shift, + tail_offst, + l.tail, + tail_size, + r.root, + r.shift, + r.tail_offset()); + l = {l.size + r.size, + concated.shift(), + concated.node(), + r.tail->inc()}; + return; } } } @@ -1064,10 +1202,9 @@ struct rrbtree // this could be improved by making sure that the // newly created nodes as part of the `push_tail()` // are tagged with `er` - auto res = l.push_tail(l.root, l.shift, tail_offst, - l.tail, tail_size); - r = { l.size + r.size, get<0>(res), get<1>(res), - r.tail->inc() }; + auto res = + l.push_tail(l.root, l.shift, tail_offst, l.tail, tail_size); + r = {l.size + r.size, get<0>(res), get<1>(res), r.tail->inc()}; return; } else if (tail_size + r.size <= branches) { // doing this in a exception way mutating way is very @@ -1077,85 +1214,116 @@ struct rrbtree // // we could however improve this by at least moving the // elements of the mutable tails... - auto new_tail = node_t::copy_leaf(l.tail, tail_size, - r.tail, r.size); - r = { l.size + r.size, l.shift, l.root->inc(), new_tail }; + auto new_tail = + node_t::copy_leaf(l.tail, tail_size, r.tail, r.size); + r = {l.size + r.size, l.shift, l.root->inc(), new_tail}; return; } else { // like the immutable version. // we could improve this also by moving elements // instead of just copying them auto remaining = branches - tail_size; - auto add_tail = node_t::copy_leaf_e(er, - l.tail, tail_size, - r.tail, remaining); - try { - auto new_tail = node_t::copy_leaf_e(er, r.tail, remaining, r.size); - try { + auto add_tail = node_t::copy_leaf_e( + er, l.tail, tail_size, r.tail, remaining); + IMMER_TRY { + auto new_tail = + node_t::copy_leaf_e(er, r.tail, remaining, r.size); + IMMER_TRY { // this could be improved by making sure that the // newly created nodes as part of the `push_tail()` // are tagged with `er` - auto new_root = l.push_tail(l.root, l.shift, tail_offst, - add_tail, branches); - r = { l.size + r.size, - get<0>(new_root), get<1>(new_root), - new_tail }; + auto new_root = l.push_tail(l.root, + l.shift, + tail_offst, + add_tail, + branches); + r = {l.size + r.size, + get<0>(new_root), + get<1>(new_root), + new_tail}; return; - } catch (...) { + } + IMMER_CATCH (...) { node_t::delete_leaf(new_tail, r.size - remaining); - throw; + IMMER_RETHROW; } - } catch (...) { + } + IMMER_CATCH (...) { node_t::delete_leaf(add_tail, branches); - throw; + IMMER_RETHROW; } - return; } } else if (l.tail_offset() == 0) { if (supports_transient_concat) { auto tail_offst = l.tail_offset(); auto tail_size = l.size - tail_offst; - auto concated = concat_trees_mut( - er, - el, l.tail, tail_size, - er,r.root, r.shift, r.tail_offset()); - assert(concated.shift() == concated.node()->compute_shift()); - assert(concated.node()->check(concated.shift(), l.size + r.tail_offset())); + auto concated = concat_trees_mut(er, + el, + l.tail, + tail_size, + er, + r.root, + r.shift, + r.tail_offset()); + IMMER_ASSERT_TAGGED(concated.shift() == + concated.node()->compute_shift()); + assert(concated.node()->check(concated.shift(), + l.size + r.tail_offset())); r.size += l.size; r.shift = concated.shift(); r.root = concated.node(); + assert(r.check_tree()); l.hard_reset(); } else { auto tail_offst = l.tail_offset(); auto tail_size = l.size - tail_offst; - auto concated = concat_trees(l.tail, tail_size, - r.root, r.shift, r.tail_offset()); - r = { l.size + r.size, concated.shift(), - concated.node(), r.tail->inc() }; + auto concated = concat_trees( + l.tail, tail_size, r.root, r.shift, r.tail_offset()); + r = {l.size + r.size, + concated.shift(), + concated.node(), + r.tail->inc()}; return; } } else { if (supports_transient_concat) { auto tail_offst = l.tail_offset(); auto tail_size = l.size - tail_offst; - auto concated = concat_trees_mut( - er, - el, l.root, l.shift, tail_offst, l.tail, tail_size, - er, r.root, r.shift, r.tail_offset()); - assert(concated.shift() == concated.node()->compute_shift()); - assert(concated.node()->check(concated.shift(), l.size + r.tail_offset())); + auto concated = concat_trees_mut(er, + el, + l.root, + l.shift, + tail_offst, + l.tail, + tail_size, + er, + r.root, + r.shift, + r.tail_offset()); + IMMER_ASSERT_TAGGED(concated.shift() == + concated.node()->compute_shift()); + assert(concated.node()->check(concated.shift(), + l.size + r.tail_offset())); r.size += l.size; r.shift = concated.shift(); r.root = concated.node(); + assert(r.check_tree()); l.hard_reset(); } else { auto tail_offst = l.tail_offset(); auto tail_size = l.size - tail_offst; - auto concated = concat_trees(l.root, l.shift, tail_offst, - l.tail, tail_size, - r.root, r.shift, r.tail_offset()); - r = { l.size + r.size, concated.shift(), - concated.node(), r.tail->inc() }; + auto concated = concat_trees(l.root, + l.shift, + tail_offst, + l.tail, + tail_size, + r.root, + r.shift, + r.tail_offset()); + r = {l.size + r.size, + concated.shift(), + concated.node(), + r.tail->inc()}; } } } @@ -1163,11 +1331,10 @@ struct rrbtree void hard_reset() { assert(supports_transient_concat); - auto&& empty_ = empty(); - size = empty_.size; - shift = empty_.shift; - root = empty_.root; - tail = empty_.tail; + size = 0; + shift = BL; + root = empty_root(); + tail = empty_tail(); } bool check_tree() const @@ -1198,7 +1365,7 @@ struct rrbtree if (tail_offset() > 0) assert(root->check(shift, tail_offset())); else { - assert(root->kind() == node_t::kind_t::inner); + IMMER_ASSERT_TAGGED(root->kind() == node_t::kind_t::inner); assert(shift == BL); } #endif @@ -1208,8 +1375,7 @@ struct rrbtree #if IMMER_DEBUG_PRINT void debug_print(std::ostream& out) const { - out - << "--" << std::endl + out << "--" << std::endl << "{" << std::endl << " size = " << size << std::endl << " shift = " << shift << std::endl @@ -1222,7 +1388,7 @@ struct rrbtree void debug_print_indent(std::ostream& out, unsigned indent) const { - while (indent --> 0) + while (indent-- > 0) out << ' '; } @@ -1237,14 +1403,12 @@ struct rrbtree if (shift == endshift) { debug_print_indent(out, indent); out << "- {" << size << "} " - << pretty_print_array(node->leaf(), size) - << std::endl; + << pretty_print_array(node->leaf(), size) << std::endl; } else if (auto r = node->relaxed()) { auto count = r->d.count; debug_print_indent(out, indent); out << "# {" << size << "} " - << pretty_print_array(r->d.sizes, r->d.count) - << std::endl; + << pretty_print_array(r->d.sizes, r->d.count) << std::endl; auto last_size = size_t{}; for (auto i = count_t{}; i < count; ++i) { debug_print_node(out, @@ -1257,8 +1421,8 @@ struct rrbtree } else { debug_print_indent(out, indent); out << "+ {" << size << "}" << std::endl; - auto count = (size >> shift) - + (size - ((size >> shift) << shift) > 0); + auto count = + (size >> shift) + (size - ((size >> shift) << shift) > 0); if (count) { for (auto i = count_t{}; i < count - 1; ++i) debug_print_node(out, diff --git a/src/immer/detail/rbts/rrbtree_iterator.hpp b/src/immer/detail/rbts/rrbtree_iterator.hpp index a40f7f486d..af967774e7 100644 --- a/src/immer/detail/rbts/rrbtree_iterator.hpp +++ b/src/immer/detail/rbts/rrbtree_iterator.hpp @@ -8,8 +8,8 @@ #pragma once -#include #include +#include namespace immer { namespace detail { @@ -27,7 +27,8 @@ struct rrbtree_iterator using tree_t = rrbtree; using region_t = std::tuple; - struct end_t {}; + struct end_t + {}; const tree_t& impl() const { return *v_; } size_t index() const { return i_; } @@ -35,23 +36,22 @@ struct rrbtree_iterator rrbtree_iterator() = default; rrbtree_iterator(const tree_t& v) - : v_ { &v } - , i_ { 0 } - , curr_ { nullptr, ~size_t{}, ~size_t{} } - { - } + : v_{&v} + , i_{0} + , curr_{nullptr, ~size_t{}, ~size_t{}} + {} rrbtree_iterator(const tree_t& v, end_t) - : v_ { &v } - , i_ { v.size } - , curr_ { nullptr, ~size_t{}, ~size_t{} } + : v_{&v} + , i_{v.size} + , curr_{nullptr, ~size_t{}, ~size_t{}} {} private: friend iterator_core_access; const tree_t* v_; - size_t i_; + size_t i_; mutable region_t curr_; void increment() @@ -76,16 +76,12 @@ struct rrbtree_iterator i_ += n; } - bool equal(const rrbtree_iterator& other) const - { - return i_ == other.i_; - } + bool equal(const rrbtree_iterator& other) const { return i_ == other.i_; } std::ptrdiff_t distance_to(const rrbtree_iterator& other) const { - return other.i_ > i_ - ? static_cast(other.i_ - i_) - : - static_cast(i_ - other.i_); + return other.i_ > i_ ? static_cast(other.i_ - i_) + : -static_cast(i_ - other.i_); } const T& dereference() const diff --git a/src/immer/detail/rbts/visitor.hpp b/src/immer/detail/rbts/visitor.hpp index eaccfcdca4..38cd030c4a 100644 --- a/src/immer/detail/rbts/visitor.hpp +++ b/src/immer/detail/rbts/visitor.hpp @@ -21,31 +21,31 @@ template struct visitor_base { template - static decltype(auto) visit_node(Args&& ...args) + static decltype(auto) visit_node(Args&&... args) { IMMER_UNREACHABLE; } template - static decltype(auto) visit_relaxed(Args&& ...args) + static decltype(auto) visit_relaxed(Args&&... args) { return Deriv::visit_inner(std::forward(args)...); } template - static decltype(auto) visit_regular(Args&& ...args) + static decltype(auto) visit_regular(Args&&... args) { return Deriv::visit_inner(std::forward(args)...); } template - static decltype(auto) visit_inner(Args&& ...args) + static decltype(auto) visit_inner(Args&&... args) { return Deriv::visit_node(std::forward(args)...); } template - static decltype(auto) visit_leaf(Args&& ...args) + static decltype(auto) visit_leaf(Args&&... args) { return Deriv::visit_node(std::forward(args)...); } diff --git a/src/immer/detail/ref_count_base.hpp b/src/immer/detail/ref_count_base.hpp index a20428b13f..28f2e46a99 100644 --- a/src/immer/detail/ref_count_base.hpp +++ b/src/immer/detail/ref_count_base.hpp @@ -16,7 +16,7 @@ namespace detail { template struct ref_count_base { - mutable std::atomic ref_count { 0 }; + mutable std::atomic ref_count{0}; friend void intrusive_ptr_add_ref(const Deriv* x) { diff --git a/src/immer/detail/type_traits.hpp b/src/immer/detail/type_traits.hpp index 7820e75d7a..091fe04dc2 100644 --- a/src/immer/detail/type_traits.hpp +++ b/src/immer/detail/type_traits.hpp @@ -18,67 +18,85 @@ namespace immer { namespace detail { template -struct make_void { using type = void; }; +struct make_void +{ + using type = void; +}; template using void_t = typename make_void::type; template -struct is_dereferenceable : std::false_type {}; +struct is_dereferenceable : std::false_type +{}; template -struct is_dereferenceable()))>> : - std::true_type {}; +struct is_dereferenceable()))>> + : std::true_type +{}; template constexpr bool is_dereferenceable_v = is_dereferenceable::value; -template -struct is_equality_comparable : std::false_type {}; +template +struct is_equality_comparable : std::false_type +{}; -template -struct is_equality_comparable -() == std::declval())>::value>> : - std::true_type {}; +template +struct is_equality_comparable< + T, + U, + std::enable_if_t() == + std::declval())>::value>> + : std::true_type +{}; -template +template constexpr bool is_equality_comparable_v = is_equality_comparable::value; -template -struct is_inequality_comparable : std::false_type {}; +template +struct is_inequality_comparable : std::false_type +{}; -template -struct is_inequality_comparable -() != std::declval())>::value>> : - std::true_type {}; +template +struct is_inequality_comparable< + T, + U, + std::enable_if_t() != + std::declval())>::value>> + : std::true_type +{}; -template +template constexpr bool is_inequality_comparable_v = - is_inequality_comparable::value; + is_inequality_comparable::value; template -struct is_preincrementable : std::false_type {}; +struct is_preincrementable : std::false_type +{}; template -struct is_preincrementable -()))>::value>> : - std::true_type {}; +struct is_preincrementable< + T, + std::enable_if_t()))>::value>> + : std::true_type +{}; template constexpr bool is_preincrementable_v = is_preincrementable::value; -template -struct is_subtractable : std::false_type {}; +template +struct is_subtractable : std::false_type +{}; template -struct is_subtractable -() - std::declval())>> : - std::true_type {}; +struct is_subtractable< + T, + U, + void_t() - std::declval())>> : std::true_type +{}; template constexpr bool is_subtractable_v = is_subtractable::value; @@ -88,104 +106,100 @@ namespace swappable { using std::swap; template -struct with : std::false_type {}; +struct with : std::false_type +{}; // Does not account for non-referenceable types template -struct with -(), std::declval())), - decltype(swap(std::declval(), std::declval()))>> : - std::true_type {}; +struct with(), std::declval())), + decltype(swap(std::declval(), std::declval()))>> + : std::true_type +{}; -} +} // namespace swappable -template +template using is_swappable_with = swappable::with; -template +template using is_swappable = is_swappable_with; template constexpr bool is_swappable_v = is_swappable_with::value; template -struct is_iterator : std::false_type {}; +struct is_iterator : std::false_type +{}; // See http://en.cppreference.com/w/cpp/concept/Iterator template -struct is_iterator - - && is_dereferenceable_v - // accounts for non-referenceable types - && std::is_copy_constructible::value - && std::is_copy_assignable::value - && std::is_destructible::value - && is_swappable_v>, - typename std::iterator_traits::value_type, - typename std::iterator_traits::difference_type, - typename std::iterator_traits::reference, - typename std::iterator_traits::pointer, - typename std::iterator_traits::iterator_category>> : - std::true_type {}; - -template +struct is_iterator< + T, + void_t< + std::enable_if_t && + is_dereferenceable_v + // accounts for non-referenceable types + && std::is_copy_constructible::value && + std::is_copy_assignable::value && + std::is_destructible::value && is_swappable_v>, + typename std::iterator_traits::value_type, + typename std::iterator_traits::difference_type, + typename std::iterator_traits::reference, + typename std::iterator_traits::pointer, + typename std::iterator_traits::iterator_category>> : std::true_type +{}; + +template constexpr bool is_iterator_v = is_iterator::value; -template -struct compatible_sentinel : std::false_type {}; - -template -struct compatible_sentinel - - && is_equality_comparable_v - && is_inequality_comparable_v>> : - std::true_type {}; - -template -constexpr bool compatible_sentinel_v = compatible_sentinel::value; - -template -struct is_forward_iterator : std::false_type {}; - -template -struct is_forward_iterator - && - std::is_base_of - ::iterator_category>::value>> : - std::true_type {}; - -template -constexpr bool is_forward_iterator_v = is_forward_iterator::value; +template +struct compatible_sentinel : std::false_type +{}; -template -struct std_distance_supports : std::false_type {}; +template +struct compatible_sentinel< + T, + U, + std::enable_if_t && is_equality_comparable_v && + is_inequality_comparable_v>> : std::true_type +{}; -template -struct std_distance_supports -(), std::declval()))>> : - std::true_type {}; +template +constexpr bool compatible_sentinel_v = compatible_sentinel::value; -template -constexpr bool std_distance_supports_v = std_distance_supports::value; +template +struct is_forward_iterator : std::false_type +{}; -template -struct std_uninitialized_copy_supports : std::false_type {}; +template +struct is_forward_iterator< + T, + std::enable_if_t && + std::is_base_of::iterator_category>::value>> + : std::true_type +{}; -template -struct std_uninitialized_copy_supports -(), - std::declval(), - std::declval()))>> : - std::true_type {}; +template +constexpr bool is_forward_iterator_v = is_forward_iterator::value; -template -constexpr bool std_uninitialized_copy_supports_v = - std_uninitialized_copy_supports::value; +template +struct std_distance_supports : std::false_type +{}; + +template +struct std_distance_supports< + T, + U, + void_t(), std::declval()))>> + : std::true_type +{}; + +template +constexpr bool std_distance_supports_v = std_distance_supports::value; -} -} +} // namespace detail +} // namespace immer diff --git a/src/immer/detail/util.hpp b/src/immer/detail/util.hpp index 854a263d9e..d6ae246b43 100644 --- a/src/immer/detail/util.hpp +++ b/src/immer/detail/util.hpp @@ -11,9 +11,9 @@ #include #include +#include #include #include -#include #include @@ -24,62 +24,171 @@ namespace immer { namespace detail { +template +const T* as_const(T* x) +{ + return x; +} + +template +const T& as_const(T& x) +{ + return x; +} + template using aligned_storage_for = typename std::aligned_storage::type; template -T& auto_const_cast(const T& x) { return const_cast(x); } +T& auto_const_cast(const T& x) +{ + return const_cast(x); +} template -T&& auto_const_cast(const T&& x) { return const_cast(std::move(x)); } +T&& auto_const_cast(const T&& x) +{ + return const_cast(std::move(x)); +} -template -auto uninitialized_move(Iter1 in1, Iter1 in2, Iter2 out) +template +inline auto destroy_at(T* p) noexcept + -> std::enable_if_t::value> { - return std::uninitialized_copy(std::make_move_iterator(in1), - std::make_move_iterator(in2), - out); + p->~T(); } template -void destroy(T* first, T* last) +inline auto destroy_at(T* p) noexcept + -> std::enable_if_t::value> +{ + p->~T(); +} + +template +constexpr bool can_trivially_detroy = std::is_trivially_destructible< + typename std::iterator_traits::value_type>::value; + +template +auto destroy(Iter, Iter last) noexcept + -> std::enable_if_t, Iter> +{ + return last; +} +template +auto destroy(Iter first, Iter last) noexcept + -> std::enable_if_t, Iter> { for (; first != last; ++first) - first->~T(); + detail::destroy_at(std::addressof(*first)); + return first; +} + +template +auto destroy_n(Iter first, Size n) noexcept + -> std::enable_if_t, Iter> +{ + return first + n; +} +template +auto destroy_n(Iter first, Size n) noexcept + -> std::enable_if_t, Iter> +{ + for (; n > 0; (void) ++first, --n) + detail::destroy_at(std::addressof(*first)); + return first; +} + +template +constexpr bool can_trivially_copy = + std::is_same::value_type, + typename std::iterator_traits::value_type>::value&& + std::is_trivially_copyable< + typename std::iterator_traits::value_type>::value; + +template +auto uninitialized_move(Iter1 first, Iter1 last, Iter2 out) noexcept + -> std::enable_if_t, Iter2> +{ + return std::copy(first, last, out); +} +template +auto uninitialized_move(Iter1 first, Iter1 last, Iter2 out) + -> std::enable_if_t, Iter2> + +{ + using value_t = typename std::iterator_traits::value_type; + auto current = out; + IMMER_TRY { + for (; first != last; ++first, (void) ++current) { + ::new (const_cast(static_cast( + std::addressof(*current)))) value_t(std::move(*first)); + } + return current; + } + IMMER_CATCH (...) { + detail::destroy(out, current); + IMMER_RETHROW; + } } -template -void destroy_n(T* p, Size n) +template +auto uninitialized_copy(SourceIter first, Sent last, SinkIter out) noexcept + -> std::enable_if_t, SinkIter> +{ + return std::copy(first, last, out); +} +template +auto uninitialized_copy(SourceIter first, Sent last, SinkIter out) + -> std::enable_if_t, SinkIter> { - auto e = p + n; - for (; p != e; ++p) - p->~T(); + using value_t = typename std::iterator_traits::value_type; + auto current = out; + IMMER_TRY { + for (; first != last; ++first, (void) ++current) { + ::new (const_cast(static_cast( + std::addressof(*current)))) value_t(*first); + } + return current; + } + IMMER_CATCH (...) { + detail::destroy(out, current); + IMMER_RETHROW; + } } template -T* make(Args&& ...args) +T* make(Args&&... args) { auto ptr = Heap::allocate(sizeof(T)); - try { + IMMER_TRY { return new (ptr) T{std::forward(args)...}; - } catch (...) { + } + IMMER_CATCH (...) { Heap::deallocate(sizeof(T), ptr); - throw; + IMMER_RETHROW; } } -struct not_supported_t {}; -struct empty_t {}; +struct not_supported_t +{}; +struct empty_t +{}; template struct exact_t { T value; - exact_t(T v) : value{v} {}; + exact_t(T v) + : value{v} {}; }; template -inline constexpr auto clz_(T) -> not_supported_t { IMMER_UNREACHABLE; return {}; } +inline constexpr auto clz_(T) -> not_supported_t +{ + IMMER_UNREACHABLE; + return {}; +} #if defined(_MSC_VER) // inline auto clz_(unsigned short x) { return __lzcnt16(x); } // inline auto clz_(unsigned int x) { return __lzcnt(x); } @@ -97,46 +206,63 @@ inline constexpr T log2_aux(T x, T r = 0) } template -inline constexpr auto log2(T x) - -> std::enable_if_t::value, T> +inline constexpr auto log2(T x) -> std:: + enable_if_t::value, T> { return x == 0 ? 0 : sizeof(std::size_t) * 8 - 1 - clz_(x); } template inline constexpr auto log2(T x) - -> std::enable_if_t::value, T> + -> std::enable_if_t::value, + T> { return log2_aux(x); } +template +constexpr T ipow(T num, unsigned int pow) +{ + return pow == 0 ? 1 : num * ipow(num, pow - 1); +} + template auto static_if(F&& f) -> std::enable_if_t -{ std::forward(f)(empty_t{}); } +{ + std::forward(f)(empty_t{}); +} template auto static_if(F&& f) -> std::enable_if_t {} -template +template auto static_if(F1&& f1, F2&& f2) -> std::enable_if_t -{ return std::forward(f1)(empty_t{}); } -template +{ + return std::forward(f1)(empty_t{}); +} +template auto static_if(F1&& f1, F2&& f2) -> std::enable_if_t -{ return std::forward(f2)(empty_t{}); } +{ + return std::forward(f2)(empty_t{}); +} template struct constantly { template - T operator() (Args&&...) const { return value; } + T operator()(Args&&...) const + { + return value; + } }; /*! * An alias to `std::distance` */ -template , bool> = true> +template , + bool> = true> typename std::iterator_traits::difference_type distance(Iterator first, Sentinel last) { @@ -147,12 +273,14 @@ distance(Iterator first, Sentinel last) * Equivalent of the `std::distance` applied to the sentinel-delimited * forward range @f$ [first, last) @f$ */ -template ) - && detail::is_forward_iterator_v - && detail::compatible_sentinel_v - && (!detail::is_subtractable_v), bool> = true> +template ) &&detail:: + is_forward_iterator_v && + detail::compatible_sentinel_v && + (!detail::is_subtractable_v), + bool> = true> typename std::iterator_traits::difference_type distance(Iterator first, Sentinel last) { @@ -168,58 +296,19 @@ distance(Iterator first, Sentinel last) * Equivalent of the `std::distance` applied to the sentinel-delimited * random access range @f$ [first, last) @f$ */ -template ) - && detail::is_forward_iterator_v - && detail::compatible_sentinel_v - && detail::is_subtractable_v, bool> = true> +template ) &&detail:: + is_forward_iterator_v && + detail::compatible_sentinel_v && + detail::is_subtractable_v, + bool> = true> typename std::iterator_traits::difference_type distance(Iterator first, Sentinel last) { return last - first; } - - -/*! - * An alias to `std::uninitialized_copy` - */ -template , bool> = true> -SinkIter uninitialized_copy(Iterator first, Sentinel last, SinkIter d_first) -{ - return std::uninitialized_copy(first, last, d_first); -} - -/*! - * Equivalent of the `std::uninitialized_copy` applied to the - * sentinel-delimited forward range @f$ [first, last) @f$ - */ -template ) - && detail::compatible_sentinel_v - && detail::is_forward_iterator_v, bool> = true> -SinkIter uninitialized_copy(SourceIter first, Sent last, SinkIter d_first) -{ - auto current = d_first; - try { - while (first != last) { - *current++ = *first; - ++first; - } - } catch (...) { - using Value = typename std::iterator_traits::value_type; - for (;d_first != current; ++d_first){ - d_first->~Value(); - } - throw; - } - return current; -} - } // namespace detail } // namespace immer diff --git a/src/immer/experimental/detail/dvektor_impl.hpp b/src/immer/experimental/detail/dvektor_impl.hpp index df273aab71..4f697337a7 100644 --- a/src/immer/experimental/detail/dvektor_impl.hpp +++ b/src/immer/experimental/detail/dvektor_impl.hpp @@ -17,6 +17,7 @@ #include #include +#include #include namespace immer { @@ -28,31 +29,32 @@ constexpr auto fast_log2(std::size_t x) return x == 0 ? 0 : sizeof(std::size_t) * 8 - 1 - __builtin_clzl(x); } -template +template constexpr T branches = T{1} << B; -template +template constexpr T mask = branches - 1; -template -constexpr auto max_depth = - fast_log2(std::numeric_limits::max()) / B; +template +constexpr auto + max_depth = fast_log2(std::numeric_limits::max()) / B; template struct node; template -using node_ptr = boost::intrusive_ptr >; +using node_ptr = boost::intrusive_ptr>; template -using leaf_node = std::array; +using leaf_node = std::array; template using inner_node = std::array, 1 << B>; template -struct node : enable_intrusive_ptr, typename MP::refcount> - , enable_optimized_heap_policy, typename MP::heap> +struct node + : enable_intrusive_ptr, typename MP::refcount> + , enable_optimized_heap_policy, typename MP::heap> { using leaf_node_t = leaf_node; using inner_node_t = inner_node; @@ -65,10 +67,14 @@ struct node : enable_intrusive_ptr, typename MP::refcount> union data_t { - leaf_node_t leaf; + leaf_node_t leaf; inner_node_t inner; - data_t(leaf_node_t n) : leaf(std::move(n)) {} - data_t(inner_node_t n) : inner(std::move(n)) {} + data_t(leaf_node_t n) + : leaf(std::move(n)) + {} + data_t(inner_node_t n) + : inner(std::move(n)) + {} ~data_t() {} } data; @@ -94,37 +100,41 @@ struct node : enable_intrusive_ptr, typename MP::refcount> , data{std::move(n)} {} - inner_node_t& inner() & { + inner_node_t& inner() & + { assert(kind == inner_kind); return data.inner; } - const inner_node_t& inner() const& { + const inner_node_t& inner() const& + { assert(kind == inner_kind); return data.inner; } - inner_node_t&& inner() && { + inner_node_t&& inner() && + { assert(kind == inner_kind); return std::move(data.inner); } - leaf_node_t& leaf() & { + leaf_node_t& leaf() & + { assert(kind == leaf_kind); return data.leaf; } - const leaf_node_t& leaf() const& { + const leaf_node_t& leaf() const& + { assert(kind == leaf_kind); return data.leaf; } - leaf_node_t&& leaf() && { + leaf_node_t&& leaf() && + { assert(kind == leaf_kind); return std::move(data.leaf); } }; -template -auto make_node(Ts&& ...xs) - -> boost::intrusive_ptr> +template +auto make_node(Ts&&... xs) -> boost::intrusive_ptr> { return new node(std::forward(xs)...); } @@ -140,8 +150,8 @@ struct ref unsigned depth; std::array> display; - template - static auto make_node(Ts&& ...xs) + template + static auto make_node(Ts&&... xs) { return dvektor::make_node(std::forward(xs)...); } @@ -152,16 +162,16 @@ struct ref auto node = display[display_idx].get(); auto shift = display_idx * B; while (display_idx--) { - node = node->inner() [(index >> shift) & mask].get(); + node = node->inner()[(index >> shift) & mask].get(); shift -= B; } - return node->leaf() [index & mask]; + return node->leaf()[index & mask]; } node_ptr_t null_slot_and_copy_inner(node_ptr_t& node, std::size_t idx) { auto& n = node->inner(); - auto x = node_ptr_t{}; + auto x = node_ptr_t{}; x.swap(n[idx]); return copy_of_inner(x); } @@ -169,7 +179,7 @@ struct ref node_ptr_t null_slot_and_copy_leaf(node_ptr_t& node, std::size_t idx) { auto& n = node->inner(); - auto x = node_ptr_t{}; + auto x = node_ptr_t{}; x.swap(n[idx]); return copy_of_leaf(x); } @@ -187,11 +197,9 @@ struct ref void stabilize(std::size_t index) { auto shift = B; - for (auto i = 1u; i < depth; ++i) - { + for (auto i = 1u; i < depth; ++i) { display[i] = copy_of_inner(display[i]); - display[i]->inner() [(index >> shift) & mask] - = display[i - 1]; + display[i]->inner()[(index >> shift) & mask] = display[i - 1]; shift += B; } } @@ -210,13 +218,11 @@ struct ref auto shift = B * d; while (--d) { display[d] = null_slot_and_copy_inner( - display[d + 1], - (index >> shift) & mask); + display[d + 1], (index >> shift) & mask); shift -= B; } - display[0] = null_slot_and_copy_leaf( - display[1], - (index >> B) & mask); + display[0] = + null_slot_and_copy_leaf(display[1], (index >> B) & mask); } } @@ -232,19 +238,17 @@ struct ref auto shift = B; for (auto i = 1u; i <= display_idx; ++i) { display[i] = copy_of_inner(display[i]); - display[i]->inner() [(old_index >> shift) & mask] - = display[i - 1]; + display[i]->inner()[(old_index >> shift) & mask] = + display[i - 1]; shift += B; } for (auto i = display_idx - 1; i > 0; --i) { shift -= B; display[i] = null_slot_and_copy_inner( - display[i + 1], - (new_index >> shift) & mask); + display[i + 1], (new_index >> shift) & mask); } - display[0] = null_slot_and_copy_leaf( - display[1], - (new_index >> B) & mask); + display[0] = + null_slot_and_copy_leaf(display[1], (new_index >> B) & mask); } } @@ -254,21 +258,18 @@ struct ref { auto display_idx = fast_log2(xr) / B; if (display_idx > 0) { - auto shift = display_idx * B; + auto shift = display_idx * B; if (display_idx == depth) { display[display_idx] = make_node(inner_t{}); - display[display_idx]->inner() - [(old_index >> shift) & mask] = + display[display_idx]->inner()[(old_index >> shift) & mask] = display[display_idx - 1]; ++depth; } while (--display_idx) { - auto node = display[display_idx + 1]->inner() - [(new_index >> shift) & mask]; - display[display_idx] = node - ? std::move(node) - : make_node(inner_t{}); - + auto node = display[display_idx + 1] + ->inner()[(new_index >> shift) & mask]; + display[display_idx] = + node ? std::move(node) : make_node(inner_t{}); } display[0] = make_node(leaf_t{}); } @@ -285,10 +286,10 @@ struct ref void goto_next_block_start(std::size_t index, std::size_t xr) { auto display_idx = fast_log2(xr) / B; - auto shift = display_idx * B; + auto shift = display_idx * B; if (display_idx > 0) { - display[display_idx - 1] = display[display_idx]->inner() - [(index >> shift) & mask]; + display[display_idx - 1] = + display[display_idx]->inner()[(index >> shift) & mask]; while (--display_idx) display[display_idx - 1] = display[display_idx]->inner()[0]; } @@ -297,11 +298,11 @@ struct ref void goto_pos(std::size_t index, std::size_t xr) { auto display_idx = fast_log2(xr) / B; - auto shift = display_idx * B; + auto shift = display_idx * B; if (display_idx) { do { - display[display_idx - 1] = display[display_idx]->inner() - [(index >> shift) & mask]; + display[display_idx - 1] = + display[display_idx]->inner()[(index >> shift) & mask]; shift -= B; } while (--display_idx); } @@ -319,11 +320,11 @@ struct impl std::size_t size; std::size_t focus; - bool dirty; - ref_t p; + bool dirty; + ref_t p; - template - static auto make_node(Ts&& ...xs) + template + static auto make_node(Ts&&... xs) { return dvektor::make_node(std::forward(xs)...); } @@ -356,23 +357,22 @@ struct impl { if (size) { auto block_index = size & ~mask; - auto lo = size & mask; + auto lo = size & mask; if (size != block_index) { - auto s = impl{ size + 1, block_index, dirty, p }; + auto s = impl{size + 1, block_index, dirty, p}; s.goto_pos_writable(focus, block_index, focus ^ block_index); - s.p.display[0]->leaf() [lo] = std::move(value); + s.p.display[0]->leaf()[lo] = std::move(value); return s; } else { - auto s = impl{ size + 1, block_index, dirty, p }; - s.goto_fresh_pos_writable(focus, block_index, focus ^ block_index); - s.p.display[0]->leaf() [lo] = std::move(value); + auto s = impl{size + 1, block_index, dirty, p}; + s.goto_fresh_pos_writable( + focus, block_index, focus ^ block_index); + s.p.display[0]->leaf()[lo] = std::move(value); return s; } } else { return impl{ - 1, 0, false, - { 1, {{ make_node(leaf_t{{std::move(value)}}) }} } - }; + 1, 0, false, {1, {{make_node(leaf_t{{std::move(value)}})}}}}; } } @@ -384,44 +384,38 @@ struct impl template impl update(std::size_t idx, FnT&& fn) const { - auto s = impl{ size, idx, dirty, p }; + auto s = impl{size, idx, dirty, p}; s.goto_pos_writable(focus, idx, focus ^ idx); - auto& v = s.p.display[0]->leaf() [idx & mask]; - v = fn(std::move(v)); + auto& v = s.p.display[0]->leaf()[idx & mask]; + v = fn(std::move(v)); return s; } impl assoc(std::size_t idx, T value) const { - return update(idx, [&] (auto&&) { - return std::move(value); - }); + return update(idx, [&](auto&&) { return std::move(value); }); } }; template -const impl empty = { - 0, - 0, - false, - ref {1, {}} -}; +const impl empty = {0, 0, false, ref{1, {}}}; template -struct iterator : boost::iterator_facade< - iterator, - T, - boost::random_access_traversal_tag, - const T&> +struct iterator + : boost::iterator_facade, + T, + boost::random_access_traversal_tag, + const T&> { - struct end_t {}; + struct end_t + {}; iterator() = default; iterator(const impl& v) - : p_{ v.p } - , i_{ 0 } - , base_{ 0 } + : p_{v.p} + , i_{0} + , base_{0} { if (v.dirty) p_.stabilize(v.focus); @@ -430,9 +424,9 @@ struct iterator : boost::iterator_facade< } iterator(const impl& v, end_t) - : p_{ v.p } - , i_{ v.size } - , base_{ (v.size-1) & ~mask } + : p_{v.p} + , i_{v.size} + , base_{(v.size - 1) & ~mask} { if (v.dirty) p_.stabilize(v.focus); @@ -445,8 +439,8 @@ struct iterator : boost::iterator_facade< using leaf_iterator = typename leaf_node::const_iterator; ref p_; - std::size_t i_; - std::size_t base_; + std::size_t i_; + std::size_t base_; leaf_iterator curr_; void increment() @@ -489,22 +483,15 @@ struct iterator : boost::iterator_facade< } } - bool equal(const iterator& other) const - { - return i_ == other.i_; - } + bool equal(const iterator& other) const { return i_ == other.i_; } std::ptrdiff_t distance_to(const iterator& other) const { - return other.i_ > i_ - ? static_cast(other.i_ - i_) - : - static_cast(i_ - other.i_); + return other.i_ > i_ ? static_cast(other.i_ - i_) + : -static_cast(i_ - other.i_); } - const T& dereference() const - { - return *curr_; - } + const T& dereference() const { return *curr_; } }; } /* namespace dvektor */ diff --git a/src/immer/experimental/dvektor.hpp b/src/immer/experimental/dvektor.hpp index 7aa0bde7b3..a4227d6498 100644 --- a/src/immer/experimental/dvektor.hpp +++ b/src/immer/experimental/dvektor.hpp @@ -9,21 +9,22 @@ #pragma once #include + #include +#include + namespace immer { -template +template class dvektor { using impl_t = detail::dvektor::impl; public: - using value_type = T; - using reference = const T&; - using size_type = std::size_t; + using value_type = T; + using reference = const T&; + using size_type = std::size_t; using difference_type = std::ptrdiff_t; using const_reference = const T&; @@ -34,29 +35,36 @@ class dvektor dvektor() = default; iterator begin() const { return {impl_}; } - iterator end() const { return {impl_, typename iterator::end_t{}}; } + iterator end() const { return {impl_, typename iterator::end_t{}}; } reverse_iterator rbegin() const { return reverse_iterator{end()}; } - reverse_iterator rend() const { return reverse_iterator{begin()}; } + reverse_iterator rend() const { return reverse_iterator{begin()}; } std::size_t size() const { return impl_.size; } bool empty() const { return impl_.size == 0; } - reference operator[] (size_type index) const - { return impl_.get(index); } + reference operator[](size_type index) const { return impl_.get(index); } dvektor push_back(value_type value) const - { return { impl_.push_back(std::move(value)) }; } + { + return {impl_.push_back(std::move(value))}; + } dvektor assoc(std::size_t idx, value_type value) const - { return { impl_.assoc(idx, std::move(value)) }; } + { + return {impl_.assoc(idx, std::move(value))}; + } template dvektor update(std::size_t idx, FnT&& fn) const - { return { impl_.update(idx, std::forward(fn)) }; } + { + return {impl_.update(idx, std::forward(fn))}; + } private: - dvektor(impl_t impl) : impl_(std::move(impl)) {} + dvektor(impl_t impl) + : impl_(std::move(impl)) + {} impl_t impl_ = detail::dvektor::empty; }; diff --git a/src/immer/flex_vector.hpp b/src/immer/flex_vector.hpp index 7ab8fcd31b..f970db910a 100644 --- a/src/immer/flex_vector.hpp +++ b/src/immer/flex_vector.hpp @@ -55,9 +55,10 @@ class flex_vector_transient; * @endrst */ template > + typename MemoryPolicy = default_memory_policy, + detail::rbts::bits_t B = default_bits, + detail::rbts::bits_t BL = + detail::rbts::derive_bits_leaf> class flex_vector { using impl_t = detail::rbts::rrbtree; @@ -66,21 +67,27 @@ class flex_vector std::integral_constant; public: - static constexpr auto bits = B; + static constexpr auto bits = B; static constexpr auto bits_leaf = BL; - using memory_policy = MemoryPolicy; + using memory_policy = MemoryPolicy; - using value_type = T; - using reference = const T&; - using size_type = detail::rbts::size_t; + using value_type = T; + using reference = const T&; + using size_type = detail::rbts::size_t; using difference_type = std::ptrdiff_t; using const_reference = const T&; - using iterator = detail::rbts::rrbtree_iterator; + using iterator = detail::rbts::rrbtree_iterator; using const_iterator = iterator; using reverse_iterator = std::reverse_iterator; - using transient_type = flex_vector_transient; + using transient_type = flex_vector_transient; + + /*! + * Returns the maximum theoretical size supported by the internal structure + * given the current B, BL. + */ + constexpr static size_type max_size() { return impl_t::max_size(); } /*! * Default constructor. It creates a flex_vector of `size() == 0`. @@ -99,9 +106,10 @@ class flex_vector * Constructs a flex_vector containing the elements in the range * defined by the input iterator `first` and range sentinel `last`. */ - template , bool> = true> + template , + bool> = true> flex_vector(Iter first, Sent last) : impl_{impl_t::from_range(first, last)} {} @@ -120,8 +128,10 @@ class flex_vector * @f$ O(1) @f$. */ flex_vector(vector v) - : impl_ { v.impl_.size, v.impl_.shift, - v.impl_.root->inc(), v.impl_.tail->inc() } + : impl_{v.impl_.size, + v.impl_.shift, + v.impl_.root->inc(), + v.impl_.tail->inc()} {} /*! @@ -129,49 +139,58 @@ class flex_vector * collection. It does not allocate memory and its complexity is * @f$ O(1) @f$. */ - iterator begin() const { return {impl_}; } + IMMER_NODISCARD iterator begin() const { return {impl_}; } /*! * Returns an iterator pointing just after the last element of the * collection. It does not allocate and its complexity is @f$ O(1) @f$. */ - iterator end() const { return {impl_, typename iterator::end_t{}}; } + IMMER_NODISCARD iterator end() const + { + return {impl_, typename iterator::end_t{}}; + } /*! * Returns an iterator that traverses the collection backwards, * pointing at the first element of the reversed collection. It * does not allocate memory and its complexity is @f$ O(1) @f$. */ - reverse_iterator rbegin() const { return reverse_iterator{end()}; } + IMMER_NODISCARD reverse_iterator rbegin() const + { + return reverse_iterator{end()}; + } /*! * Returns an iterator that traverses the collection backwards, * pointing after the last element of the reversed collection. It * does not allocate memory and its complexity is @f$ O(1) @f$. */ - reverse_iterator rend() const { return reverse_iterator{begin()}; } + IMMER_NODISCARD reverse_iterator rend() const + { + return reverse_iterator{begin()}; + } /*! * Returns the number of elements in the container. It does * not allocate memory and its complexity is @f$ O(1) @f$. */ - size_type size() const { return impl_.size; } + IMMER_NODISCARD size_type size() const { return impl_.size; } /*! * Returns `true` if there are no elements in the container. It * does not allocate memory and its complexity is @f$ O(1) @f$. */ - bool empty() const { return impl_.size == 0; } + IMMER_NODISCARD bool empty() const { return impl_.size == 0; } /*! * Access the last element. */ - const T& back() const { return impl_.back(); } + IMMER_NODISCARD const T& back() const { return impl_.back(); } /*! * Access the first element. */ - const T& front() const { return impl_.front(); } + IMMER_NODISCARD const T& front() const { return impl_.front(); } /*! * Returns a `const` reference to the element at position `index`. @@ -179,8 +198,10 @@ class flex_vector * allocate memory and its complexity is *effectively* @f$ O(1) * @f$. */ - reference operator[] (size_type index) const - { return impl_.get(index); } + IMMER_NODISCARD reference operator[](size_type index) const + { + return impl_.get(index); + } /*! * Returns a `const` reference to the element at position @@ -188,16 +209,19 @@ class flex_vector * index \geq size() @f$. It does not allocate memory and its * complexity is *effectively* @f$ O(1) @f$. */ - reference at(size_type index) const - { return impl_.get_check(index); } + reference at(size_type index) const { return impl_.get_check(index); } /*! * Returns whether the vectors are equal. */ - bool operator==(const flex_vector& other) const - { return impl_.equals(other.impl_); } - bool operator!=(const flex_vector& other) const - { return !(*this == other); } + IMMER_NODISCARD bool operator==(const flex_vector& other) const + { + return impl_.equals(other.impl_); + } + IMMER_NODISCARD bool operator!=(const flex_vector& other) const + { + return !(*this == other); + } /*! * Returns a flex_vector with `value` inserted at the end. It may @@ -214,14 +238,18 @@ class flex_vector * * @endrst */ - flex_vector push_back(value_type value) const& - { return impl_.push_back(std::move(value)); } + IMMER_NODISCARD flex_vector push_back(value_type value) const& + { + return impl_.push_back(std::move(value)); + } - decltype(auto) push_back(value_type value) && - { return push_back_move(move_t{}, std::move(value)); } + IMMER_NODISCARD decltype(auto) push_back(value_type value) && + { + return push_back_move(move_t{}, std::move(value)); + } /*! - * Returns a flex_vector with `value` inserted at the frony. It may + * Returns a flex_vector with `value` inserted at the front. It may * allocate memory and its complexity is @f$ O(log(size)) @f$. * * @rst @@ -235,8 +263,10 @@ class flex_vector * * @endrst */ - flex_vector push_front(value_type value) const - { return flex_vector{}.push_back(value) + *this; } + IMMER_NODISCARD flex_vector push_front(value_type value) const + { + return flex_vector{}.push_back(value) + *this; + } /*! * Returns a flex_vector containing value `value` at position `index`. @@ -255,11 +285,15 @@ class flex_vector * * @endrst */ - flex_vector set(size_type index, value_type value) const& - { return impl_.assoc(index, std::move(value)); } + IMMER_NODISCARD flex_vector set(size_type index, value_type value) const& + { + return impl_.assoc(index, std::move(value)); + } - decltype(auto) set(size_type index, value_type value) && - { return set_move(move_t{}, index, std::move(value)); } + IMMER_NODISCARD decltype(auto) set(size_type index, value_type value) && + { + return set_move(move_t{}, index, std::move(value)); + } /*! * Returns a vector containing the result of the expression @@ -281,12 +315,16 @@ class flex_vector */ template - flex_vector update(size_type index, FnT&& fn) const& - { return impl_.update(index, std::forward(fn)); } + IMMER_NODISCARD flex_vector update(size_type index, FnT&& fn) const& + { + return impl_.update(index, std::forward(fn)); + } template - decltype(auto) update(size_type index, FnT&& fn) && - { return update_move(move_t{}, index, std::forward(fn)); } + IMMER_NODISCARD decltype(auto) update(size_type index, FnT&& fn) && + { + return update_move(move_t{}, index, std::forward(fn)); + } /*! * Returns a vector containing only the first `min(elems, size())` @@ -304,11 +342,15 @@ class flex_vector * * @endrst */ - flex_vector take(size_type elems) const& - { return impl_.take(elems); } + IMMER_NODISCARD flex_vector take(size_type elems) const& + { + return impl_.take(elems); + } - decltype(auto) take(size_type elems) && - { return take_move(move_t{}, elems); } + IMMER_NODISCARD decltype(auto) take(size_type elems) && + { + return take_move(move_t{}, elems); + } /*! * Returns a vector without the first `min(elems, size())` @@ -326,11 +368,15 @@ class flex_vector * * @endrst */ - flex_vector drop(size_type elems) const& - { return impl_.drop(elems); } + IMMER_NODISCARD flex_vector drop(size_type elems) const& + { + return impl_.drop(elems); + } - decltype(auto) drop(size_type elems) && - { return drop_move(move_t{}, elems); } + IMMER_NODISCARD decltype(auto) drop(size_type elems) && + { + return drop_move(move_t{}, elems); + } /*! * Concatenation operator. Returns a flex_vector with the contents @@ -348,17 +394,29 @@ class flex_vector * * @endrst */ - friend flex_vector operator+ (const flex_vector& l, const flex_vector& r) - { return l.impl_.concat(r.impl_); } + IMMER_NODISCARD friend flex_vector operator+(const flex_vector& l, + const flex_vector& r) + { + return l.impl_.concat(r.impl_); + } - friend decltype(auto) operator+ (flex_vector&& l, const flex_vector& r) - { return concat_move(move_t{}, std::move(l), r); } + IMMER_NODISCARD friend decltype(auto) operator+(flex_vector&& l, + const flex_vector& r) + { + return concat_move(move_t{}, std::move(l), r); + } - friend decltype(auto) operator+ (const flex_vector& l, flex_vector&& r) - { return concat_move(move_t{}, l, std::move(r)); } + IMMER_NODISCARD friend decltype(auto) operator+(const flex_vector& l, + flex_vector&& r) + { + return concat_move(move_t{}, l, std::move(r)); + } - friend decltype(auto) operator+ (flex_vector&& l, flex_vector&& r) - { return concat_move(move_t{}, std::move(l), std::move(r)); } + IMMER_NODISCARD friend decltype(auto) operator+(flex_vector&& l, + flex_vector&& r) + { + return concat_move(move_t{}, std::move(l), std::move(r)); + } /*! * Returns a flex_vector with the `value` inserted at index @@ -376,19 +434,23 @@ class flex_vector * * @endrst */ - flex_vector insert(size_type pos, T value) const& - { return take(pos).push_back(std::move(value)) + drop(pos); } - decltype(auto) insert(size_type pos, T value) && + IMMER_NODISCARD flex_vector insert(size_type pos, T value) const& + { + return take(pos).push_back(std::move(value)) + drop(pos); + } + IMMER_NODISCARD decltype(auto) insert(size_type pos, T value) && { using std::move; auto rs = drop(pos); - return std::move(*this).take(pos).push_back( - std::move(value)) + std::move(rs); + return std::move(*this).take(pos).push_back(std::move(value)) + + std::move(rs); } - flex_vector insert(size_type pos, flex_vector value) const& - { return take(pos) + std::move(value) + drop(pos); } - decltype(auto) insert(size_type pos, flex_vector value) && + IMMER_NODISCARD flex_vector insert(size_type pos, flex_vector value) const& + { + return take(pos) + std::move(value) + drop(pos); + } + IMMER_NODISCARD decltype(auto) insert(size_type pos, flex_vector value) && { using std::move; auto rs = drop(pos); @@ -410,17 +472,21 @@ class flex_vector * * @endrst */ - flex_vector erase(size_type pos) const& - { return take(pos) + drop(pos + 1); } - decltype(auto) erase(size_type pos) && + IMMER_NODISCARD flex_vector erase(size_type pos) const& + { + return take(pos) + drop(pos + 1); + } + IMMER_NODISCARD decltype(auto) erase(size_type pos) && { auto rs = drop(pos + 1); return std::move(*this).take(pos) + std::move(rs); } - flex_vector erase(size_type pos, size_type lpos) const& - { return lpos > pos ? take(pos) + drop(lpos) : *this; } - decltype(auto) erase(size_type pos, size_type lpos) && + IMMER_NODISCARD flex_vector erase(size_type pos, size_type lpos) const& + { + return lpos > pos ? take(pos) + drop(lpos) : *this; + } + IMMER_NODISCARD decltype(auto) erase(size_type pos, size_type lpos) && { if (lpos > pos) { auto rs = drop(lpos); @@ -434,17 +500,28 @@ class flex_vector * Returns an @a transient form of this container, an * `immer::flex_vector_transient`. */ - transient_type transient() const& - { return transient_type{ impl_ }; } - transient_type transient() && - { return transient_type{ std::move(impl_) }; } + IMMER_NODISCARD transient_type transient() const& { return impl_; } + IMMER_NODISCARD transient_type transient() && { return std::move(impl_); } + + /*! + * Returns a value that can be used as identity for the container. If two + * values have the same identity, they are guaranteed to be equal and to + * contain the same objects. However, two equal containers are not + * guaranteed to have the same identity. + */ + std::pair identity() const + { + return {impl_.root, impl_.tail}; + } // Semi-private const impl_t& impl() const { return impl_; } #if IMMER_DEBUG_PRINT - void debug_print(std::ostream& out=std::cerr) const - { impl_.debug_print(out); } + void debug_print(std::ostream& out = std::cerr) const + { + impl_.debug_print(out); + } #endif private: @@ -456,47 +533,92 @@ class flex_vector #if IMMER_DEBUG_PRINT // force the compiler to generate debug_print, so we can call // it from a debugger - [](volatile auto){}(&flex_vector::debug_print); + [](volatile auto) {}(&flex_vector::debug_print); #endif } flex_vector&& push_back_move(std::true_type, value_type value) - { impl_.push_back_mut({}, std::move(value)); return std::move(*this); } + { + impl_.push_back_mut({}, std::move(value)); + return std::move(*this); + } flex_vector push_back_move(std::false_type, value_type value) - { return impl_.push_back(std::move(value)); } + { + return impl_.push_back(std::move(value)); + } flex_vector&& set_move(std::true_type, size_type index, value_type value) - { impl_.assoc_mut({}, index, std::move(value)); return std::move(*this); } + { + impl_.assoc_mut({}, index, std::move(value)); + return std::move(*this); + } flex_vector set_move(std::false_type, size_type index, value_type value) - { return impl_.assoc(index, std::move(value)); } + { + return impl_.assoc(index, std::move(value)); + } template flex_vector&& update_move(std::true_type, size_type index, Fn&& fn) - { impl_.update_mut({}, index, std::forward(fn)); return std::move(*this); } + { + impl_.update_mut({}, index, std::forward(fn)); + return std::move(*this); + } template flex_vector update_move(std::false_type, size_type index, Fn&& fn) - { return impl_.update(index, std::forward(fn)); } + { + return impl_.update(index, std::forward(fn)); + } flex_vector&& take_move(std::true_type, size_type elems) - { impl_.take_mut({}, elems); return std::move(*this); } + { + impl_.take_mut({}, elems); + return std::move(*this); + } flex_vector take_move(std::false_type, size_type elems) - { return impl_.take(elems); } + { + return impl_.take(elems); + } flex_vector&& drop_move(std::true_type, size_type elems) - { impl_.drop_mut({}, elems); return std::move(*this); } + { + impl_.drop_mut({}, elems); + return std::move(*this); + } flex_vector drop_move(std::false_type, size_type elems) - { return impl_.drop(elems); } - - static flex_vector&& concat_move(std::true_type, flex_vector&& l, const flex_vector& r) - { concat_mut_l(l.impl_, {}, r.impl_); return std::move(l); } - static flex_vector&& concat_move(std::true_type, const flex_vector& l, flex_vector&& r) - { concat_mut_r(l.impl_, r.impl_, {}); return std::move(r); } - static flex_vector&& concat_move(std::true_type, flex_vector&& l, flex_vector&& r) - { concat_mut_lr_l(l.impl_, {}, r.impl_, {}); return std::move(l); } - static flex_vector concat_move(std::false_type, const flex_vector& l, const flex_vector& r) - { return l.impl_.concat(r.impl_); } - - impl_t impl_ = impl_t::empty(); + { + return impl_.drop(elems); + } + + static flex_vector&& + concat_move(std::true_type, flex_vector&& l, const flex_vector& r) + { + concat_mut_l(l.impl_, {}, r.impl_); + return std::move(l); + } + static flex_vector&& + concat_move(std::true_type, const flex_vector& l, flex_vector&& r) + { + concat_mut_r(l.impl_, r.impl_, {}); + return std::move(r); + } + static flex_vector&& + concat_move(std::true_type, flex_vector&& l, flex_vector&& r) + { + concat_mut_lr_l(l.impl_, {}, r.impl_, {}); + return std::move(l); + } + static flex_vector + concat_move(std::false_type, const flex_vector& l, const flex_vector& r) + { + return l.impl_.concat(r.impl_); + } + + impl_t impl_ = {}; }; +static_assert(std::is_nothrow_move_constructible>::value, + "flex_vector is not nothrow move constructible"); +static_assert(std::is_nothrow_move_assignable>::value, + "flex_vector is not nothrow move assignable"); + } // namespace immer diff --git a/src/immer/flex_vector_transient.hpp b/src/immer/flex_vector_transient.hpp index 0d6e50aae1..5c419ae02b 100644 --- a/src/immer/flex_vector_transient.hpp +++ b/src/immer/flex_vector_transient.hpp @@ -37,32 +37,32 @@ class vector_transient; * @endrst */ template > -class flex_vector_transient - : MemoryPolicy::transience_t::owner + typename MemoryPolicy = default_memory_policy, + detail::rbts::bits_t B = default_bits, + detail::rbts::bits_t BL = + detail::rbts::derive_bits_leaf> +class flex_vector_transient : MemoryPolicy::transience_t::owner { - using impl_t = detail::rbts::rrbtree; - using base_t = typename MemoryPolicy::transience_t::owner; + using impl_t = detail::rbts::rrbtree; + using base_t = typename MemoryPolicy::transience_t::owner; using owner_t = typename MemoryPolicy::transience_t::owner; public: - static constexpr auto bits = B; + static constexpr auto bits = B; static constexpr auto bits_leaf = BL; - using memory_policy = MemoryPolicy; + using memory_policy = MemoryPolicy; - using value_type = T; - using reference = const T&; - using size_type = detail::rbts::size_t; + using value_type = T; + using reference = const T&; + using size_type = detail::rbts::size_t; using difference_type = std::ptrdiff_t; using const_reference = const T&; - using iterator = detail::rbts::rrbtree_iterator; + using iterator = detail::rbts::rrbtree_iterator; using const_iterator = iterator; using reverse_iterator = std::reverse_iterator; - using persistent_type = flex_vector; + using persistent_type = flex_vector; /*! * Default constructor. It creates a flex_vector of `size() == 0`. It @@ -76,9 +76,11 @@ class flex_vector_transient * @f$ O(1) @f$. */ flex_vector_transient(vector_transient v) - : base_t { std::move(static_cast(v)) } - , impl_ { v.impl_.size, v.impl_.shift, - v.impl_.root->inc(), v.impl_.tail->inc() } + : base_t{std::move(static_cast(v))} + , impl_{v.impl_.size, + v.impl_.shift, + v.impl_.root->inc(), + v.impl_.tail->inc()} {} /*! @@ -86,39 +88,48 @@ class flex_vector_transient * collection. It does not allocate memory and its complexity is * @f$ O(1) @f$. */ - iterator begin() const { return {impl_}; } + IMMER_NODISCARD iterator begin() const { return {impl_}; } /*! * Returns an iterator pointing just after the last element of the * collection. It does not allocate and its complexity is @f$ O(1) @f$. */ - iterator end() const { return {impl_, typename iterator::end_t{}}; } + IMMER_NODISCARD iterator end() const + { + return {impl_, typename iterator::end_t{}}; + } /*! * Returns an iterator that traverses the collection backwards, * pointing at the first element of the reversed collection. It * does not allocate memory and its complexity is @f$ O(1) @f$. */ - reverse_iterator rbegin() const { return reverse_iterator{end()}; } + IMMER_NODISCARD reverse_iterator rbegin() const + { + return reverse_iterator{end()}; + } /*! * Returns an iterator that traverses the collection backwards, * pointing after the last element of the reversed collection. It * does not allocate memory and its complexity is @f$ O(1) @f$. */ - reverse_iterator rend() const { return reverse_iterator{begin()}; } + IMMER_NODISCARD reverse_iterator rend() const + { + return reverse_iterator{begin()}; + } /*! * Returns the number of elements in the container. It does * not allocate memory and its complexity is @f$ O(1) @f$. */ - size_type size() const { return impl_.size; } + IMMER_NODISCARD size_type size() const { return impl_.size; } /*! * Returns `true` if there are no elements in the container. It * does not allocate memory and its complexity is @f$ O(1) @f$. */ - bool empty() const { return impl_.size == 0; } + IMMER_NODISCARD bool empty() const { return impl_.size == 0; } /*! * Returns a `const` reference to the element at position `index`. @@ -126,8 +137,7 @@ class flex_vector_transient * allocate memory and its complexity is *effectively* @f$ O(1) * @f$. */ - reference operator[] (size_type index) const - { return impl_.get(index); } + reference operator[](size_type index) const { return impl_.get(index); } /*! * Returns a `const` reference to the element at position @@ -135,15 +145,16 @@ class flex_vector_transient * index \geq size() @f$. It does not allocate memory and its * complexity is *effectively* @f$ O(1) @f$. */ - reference at(size_type index) const - { return impl_.get_check(index); } + reference at(size_type index) const { return impl_.get_check(index); } /*! * Inserts `value` at the end. It may allocate memory and its * complexity is *effectively* @f$ O(1) @f$. */ void push_back(value_type value) - { impl_.push_back_mut(*this, std::move(value)); } + { + impl_.push_back_mut(*this, std::move(value)); + } /*! * Sets to the value `value` at position `idx`. @@ -152,7 +163,9 @@ class flex_vector_transient * *effectively* @f$ O(1) @f$. */ void set(size_type index, value_type value) - { impl_.assoc_mut(*this, index, std::move(value)); } + { + impl_.assoc_mut(*this, index, std::move(value)); + } /*! * Updates the vector to contain the result of the expression @@ -163,35 +176,23 @@ class flex_vector_transient */ template void update(size_type index, FnT&& fn) - { impl_.update_mut(*this, index, std::forward(fn)); } + { + impl_.update_mut(*this, index, std::forward(fn)); + } /*! * Resizes the vector to only contain the first `min(elems, size())` * elements. It may allocate memory and its complexity is * *effectively* @f$ O(1) @f$. */ - void take(size_type elems) - { impl_.take_mut(*this, elems); } + void take(size_type elems) { impl_.take_mut(*this, elems); } /*! * Removes the first the first `min(elems, size())` * elements. It may allocate memory and its complexity is * *effectively* @f$ O(1) @f$. */ - void drop(size_type elems) - { impl_.drop_mut(*this, elems); } - - /*! - * Returns an @a immutable form of this container, an - * `immer::flex_vector`. - */ - persistent_type persistent() & - { - this->owner_t::operator=(owner_t{}); - return persistent_type{ impl_ }; - } - persistent_type persistent() && - { return persistent_type{ std::move(impl_) }; } + void drop(size_type elems) { impl_.drop_mut(*this, elems); } /*! * Appends the contents of the `r` at the end. It may allocate @@ -204,7 +205,9 @@ class flex_vector_transient concat_mut_l(impl_, *this, r.impl_); } void append(flex_vector_transient&& r) - { concat_mut_lr_l(impl_, *this, r.impl_, r); } + { + concat_mut_lr_l(impl_, *this, r.impl_, r); + } /*! * Prepends the contents of the `l` at the beginning. It may @@ -217,7 +220,20 @@ class flex_vector_transient concat_mut_r(l.impl_, impl_, *this); } void prepend(flex_vector_transient&& l) - { concat_mut_lr_r(l.impl_, l, impl_, *this); } + { + concat_mut_lr_r(l.impl_, l, impl_, *this); + } + + /*! + * Returns an @a immutable form of this container, an + * `immer::flex_vector`. + */ + IMMER_NODISCARD persistent_type persistent() & + { + this->owner_t::operator=(owner_t{}); + return impl_; + } + IMMER_NODISCARD persistent_type persistent() && { return std::move(impl_); } private: friend persistent_type; @@ -226,7 +242,7 @@ class flex_vector_transient : impl_(std::move(impl)) {} - impl_t impl_ = impl_t::empty(); + impl_t impl_ = {}; }; } // namespace immer diff --git a/src/immer/heap/cpp_heap.hpp b/src/immer/heap/cpp_heap.hpp index cd129b406b..3789754821 100644 --- a/src/immer/heap/cpp_heap.hpp +++ b/src/immer/heap/cpp_heap.hpp @@ -8,6 +8,7 @@ #pragma once +#include #include namespace immer { @@ -32,7 +33,7 @@ struct cpp_heap * `allocate`. One must not use nor deallocate again a memory * region that once it has been deallocated. */ - static void deallocate(std::size_t size, void* data) + static void deallocate(std::size_t, void* data) { ::operator delete(data); } diff --git a/src/immer/heap/debug_size_heap.hpp b/src/immer/heap/debug_size_heap.hpp index 5cfb74ca8f..d5288c646f 100644 --- a/src/immer/heap/debug_size_heap.hpp +++ b/src/immer/heap/debug_size_heap.hpp @@ -8,11 +8,14 @@ #pragma once -#include -#include #include #include +#include +#include +#include +#include + namespace immer { #if IMMER_ENABLE_DEBUG_SIZE_HEAP @@ -24,15 +27,27 @@ namespace immer { template struct debug_size_heap { - // temporary fix until https://github.com/arximboldi/immer/issues/78 is fixed - constexpr static auto extra_size = sizeof(void*) * 2; //alignof(std::max_align_t); +#if defined(__MINGW32__) && !defined(__MINGW64__) + // There is a bug in MinGW 32bit: + // https://sourceforge.net/p/mingw-w64/bugs/778/ It causes different + // versions of std::max_align_t to be defined, depending on inclusion order + // of stddef.h and stdint.h. As we have no control over the inclusion order + // here (as it might be set in stone by the outside world), we can't easily + // pin it to one of both versions of std::max_align_t. This means, we have + // to hardcode extra_size for MinGW 32bit builds until the mentioned bug is + // fixed. + constexpr static auto extra_size = 8; +#else + constexpr static auto extra_size = sizeof( + std::aligned_storage_t); +#endif template static void* allocate(std::size_t size, Tags... tags) { auto p = (std::size_t*) Base::allocate(size + extra_size, tags...); - *p = size; - return ((char*)p) + extra_size; + new (p) std::size_t{size}; + return ((char*) p) + extra_size; } template diff --git a/src/immer/heap/free_list_heap.hpp b/src/immer/heap/free_list_heap.hpp index d82d0967b0..15ae01d60f 100644 --- a/src/immer/heap/free_list_heap.hpp +++ b/src/immer/heap/free_list_heap.hpp @@ -8,11 +8,12 @@ #pragma once -#include #include +#include #include #include +#include namespace immer { diff --git a/src/immer/heap/free_list_node.hpp b/src/immer/heap/free_list_node.hpp index 5c2f5b7529..acab4779aa 100644 --- a/src/immer/heap/free_list_node.hpp +++ b/src/immer/heap/free_list_node.hpp @@ -18,8 +18,7 @@ struct free_list_node }; template -struct with_free_list_node - : with_data +struct with_free_list_node : with_data {}; } // namespace immer diff --git a/src/immer/heap/gc_heap.hpp b/src/immer/heap/gc_heap.hpp index f37da2f6e3..8fc7754e5e 100644 --- a/src/immer/heap/gc_heap.hpp +++ b/src/immer/heap/gc_heap.hpp @@ -17,30 +17,34 @@ #error "Using garbage collection requires libgc" #endif +#include #include +#include #include namespace immer { -#ifdef __APPLE__ -#define IMMER_GC_REQUIRE_INIT 1 +#ifdef IMMER_GC_REQUIRE_INIT +#define IMMER_GC_REQUIRE_INIT_ IMMER_GC_REQUIRE_INIT +#elifdef __APPLE__ +#define IMMER_GC_REQUIRE_INIT_ 1 #else -#define IMMER_GC_REQUIRE_INIT 0 +#define IMMER_GC_REQUIRE_INIT_ 0 #endif -#if IMMER_GC_REQUIRE_INIT +#if IMMER_GC_REQUIRE_INIT_ namespace detail { -template +template struct gc_initializer { - gc_initializer() { GC_init(); } + gc_initializer() { GC_INIT(); } static gc_initializer init; }; template -gc_initializer gc_initializer::init {}; +gc_initializer gc_initializer::init{}; inline void gc_initializer_guard() { @@ -56,7 +60,7 @@ inline void gc_initializer_guard() #define IMMER_GC_INIT_GUARD_ -#endif // IMMER_GC_REQUIRE_INIT +#endif // IMMER_GC_REQUIRE_INIT_ /*! * Heap that uses a tracing garbage collector. @@ -103,7 +107,7 @@ class gc_heap IMMER_GC_INIT_GUARD_; auto p = GC_malloc(n); if (IMMER_UNLIKELY(!p)) - throw std::bad_alloc{}; + IMMER_THROW(std::bad_alloc{}); return p; } @@ -112,14 +116,11 @@ class gc_heap IMMER_GC_INIT_GUARD_; auto p = GC_malloc_atomic(n); if (IMMER_UNLIKELY(!p)) - throw std::bad_alloc{}; + IMMER_THROW(std::bad_alloc{}); return p; } - static void deallocate(std::size_t, void* data) - { - GC_free(data); - } + static void deallocate(std::size_t, void* data) { GC_free(data); } static void deallocate(std::size_t, void* data, norefs_tag) { diff --git a/src/immer/heap/heap_policy.hpp b/src/immer/heap/heap_policy.hpp index a0723cbcf2..1a858c5e12 100644 --- a/src/immer/heap/heap_policy.hpp +++ b/src/immer/heap/heap_policy.hpp @@ -8,14 +8,14 @@ #pragma once +#include #include #include #include #include -#include -#include #include +#include namespace immer { @@ -37,18 +37,18 @@ struct heap_policy template struct enable_optimized_heap_policy { - static void* operator new (std::size_t size) + static void* operator new(std::size_t size) { - using heap_type = typename HeapPolicy - ::template optimized::type; + using heap_type = + typename HeapPolicy ::template optimized::type; return heap_type::allocate(size); } - static void operator delete (void* data, std::size_t size) + static void operator delete(void* data, std::size_t size) { - using heap_type = typename HeapPolicy - ::template optimized::type; + using heap_type = + typename HeapPolicy ::template optimized::type; heap_type::deallocate(size, data); } @@ -85,8 +85,7 @@ struct enable_optimized_heap_policy * @rst * * .. tip:: For many applications that use immutable data structures - * significantly, this is actually the best heap policy, and it - * might become the default in the future. + * significantly, this is actually the best heap policy. * * Note that most our data structures internally use trees with the * same big branching factors. This means that all *vectors*, @@ -99,8 +98,7 @@ struct enable_optimized_heap_policy * * @endrst */ -template +template struct free_list_heap_policy { using type = debug_size_heap; @@ -108,16 +106,13 @@ struct free_list_heap_policy template struct optimized { - using type = split_heap< - Size, - with_free_list_node< - thread_local_free_list_heap< - Size, - Limit, - free_list_heap< - Size, Limit, - debug_size_heap>>>, - debug_size_heap>; + using type = + split_heap>>>, + debug_size_heap>; }; }; @@ -126,8 +121,7 @@ struct free_list_heap_policy * multi-threading, so a single global free list with no concurrency * checks is used. */ -template +template struct unsafe_free_list_heap_policy { using type = Heap; @@ -138,9 +132,7 @@ struct unsafe_free_list_heap_policy using type = split_heap< Size, with_free_list_node< - unsafe_free_list_heap< - Size, Limit, - debug_size_heap>>, + unsafe_free_list_heap>>, debug_size_heap>; }; }; diff --git a/src/immer/heap/malloc_heap.hpp b/src/immer/heap/malloc_heap.hpp index 73909058de..af9e983fae 100644 --- a/src/immer/heap/malloc_heap.hpp +++ b/src/immer/heap/malloc_heap.hpp @@ -10,8 +10,10 @@ #include -#include +#include #include +#include +#include namespace immer { @@ -29,7 +31,7 @@ struct malloc_heap { auto p = std::malloc(size); if (IMMER_UNLIKELY(!p)) - throw std::bad_alloc{}; + IMMER_THROW(std::bad_alloc{}); return p; } @@ -38,10 +40,7 @@ struct malloc_heap * `allocate`. One must not use nor deallocate again a memory * region that once it has been deallocated. */ - static void deallocate(std::size_t, void* data) - { - std::free(data); - } + static void deallocate(std::size_t, void* data) { std::free(data); } }; } // namespace immer diff --git a/src/immer/heap/split_heap.hpp b/src/immer/heap/split_heap.hpp index 8ce210815f..18ae684bfd 100644 --- a/src/immer/heap/split_heap.hpp +++ b/src/immer/heap/split_heap.hpp @@ -10,6 +10,7 @@ #include #include +#include namespace immer { @@ -23,9 +24,8 @@ struct split_heap template static void* allocate(std::size_t size, Tags... tags) { - return size <= Size - ? SmallHeap::allocate(size, tags...) - : BigHeap::allocate(size, tags...); + return size <= Size ? SmallHeap::allocate(size, tags...) + : BigHeap::allocate(size, tags...); } template diff --git a/src/immer/heap/tags.hpp b/src/immer/heap/tags.hpp index a3012bd3ef..d1ce48d05c 100644 --- a/src/immer/heap/tags.hpp +++ b/src/immer/heap/tags.hpp @@ -10,6 +10,7 @@ namespace immer { -struct norefs_tag {}; +struct norefs_tag +{}; } // namespace immer diff --git a/src/immer/heap/thread_local_free_list_heap.hpp b/src/immer/heap/thread_local_free_list_heap.hpp index 2539ce73c1..b97a7f5082 100644 --- a/src/immer/heap/thread_local_free_list_heap.hpp +++ b/src/immer/heap/thread_local_free_list_heap.hpp @@ -10,6 +10,8 @@ #include +#include + namespace immer { namespace detail { @@ -45,11 +47,11 @@ struct thread_local_free_list_storage * @tparam Base Type of the parent heap. */ template -struct thread_local_free_list_heap : detail::unsafe_free_list_heap_impl< - detail::thread_local_free_list_storage, - Size, - Limit, - Base> +struct thread_local_free_list_heap + : detail::unsafe_free_list_heap_impl {}; } // namespace immer diff --git a/src/immer/heap/unsafe_free_list_heap.hpp b/src/immer/heap/unsafe_free_list_heap.hpp index 9a1fdd73e4..9b41dfcb37 100644 --- a/src/immer/heap/unsafe_free_list_heap.hpp +++ b/src/immer/heap/unsafe_free_list_heap.hpp @@ -10,7 +10,9 @@ #include #include + #include +#include namespace immer { namespace detail { @@ -26,12 +28,12 @@ struct unsafe_free_list_storage static head_t& head() { - static head_t head_ {nullptr, 0}; + static head_t head_{nullptr, 0}; return head_; } }; -template class Storage, +template