Skip to content

Commit

Permalink
fix: Size calculation and load/store of vectors with aligned types
Browse files Browse the repository at this point in the history
  • Loading branch information
@Tradias
Tradias committed Oct 3, 2024
1 parent b8cb102 commit 23c1df0
Show file tree
Hide file tree
Showing 14 changed files with 338 additions and 161 deletions.
2 changes: 1 addition & 1 deletion src/cntgs/detail/array.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ constexpr auto convert_array_to_size(const detail::Array<T, K>& array, std::inde
template <std::size_t N, class T, std::size_t K>
constexpr auto convert_array_to_size(const detail::Array<T, K>& array)
{
return detail::convert_array_to_size<N>(array, std::make_index_sequence<std::min(N, K)>{});
return detail::convert_array_to_size<N>(array, std::make_index_sequence<(std::min)(N, K)>{});
}
} // namespace cntgs::detail

Expand Down
25 changes: 9 additions & 16 deletions src/cntgs/detail/elementLocator.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
#define CNTGS_DETAIL_ELEMENTLOCATOR_HPP

#include "cntgs/detail/elementTraits.hpp"
#include "cntgs/detail/memory.hpp"
#include "cntgs/detail/parameterListTraits.hpp"
#include "cntgs/detail/typeTraits.hpp"
#include "cntgs/detail/utility.hpp"
Expand Down Expand Up @@ -176,10 +175,11 @@ class ElementLocator : public BaseElementLocator<Allocator>
trivially_copy_into(this->last_element_, old_memory_begin, new_memory_begin);
}

static constexpr auto calculate_new_memory_size(std::size_t max_element_count, std::size_t varying_size_bytes,
const FixedSizesArray& fixed_sizes) noexcept
static constexpr std::size_t calculate_new_memory_size(std::size_t max_element_count,
std::size_t varying_size_bytes,
const FixedSizesArray& fixed_sizes) noexcept
{
return varying_size_bytes + ElementTraits::calculate_element_size(fixed_sizes) * max_element_count;
return ElementTraits::calculate_needed_memory_size(max_element_count, varying_size_bytes, fixed_sizes);
}

private:
Expand Down Expand Up @@ -252,8 +252,7 @@ class AllFixedSizeElementLocator : public BaseAllFixedSizeElementLocator
template <class Allocator>
constexpr AllFixedSizeElementLocator(std::size_t, std::byte* memory_begin, const FixedSizesArray& fixed_sizes,
const Allocator&) noexcept
: BaseAllFixedSizeElementLocator({}, ElementTraits::calculate_element_size(fixed_sizes),
AllFixedSizeElementLocator::calculate_element_start(memory_begin))
: BaseAllFixedSizeElementLocator({}, ElementTraits::calculate_element_stride(fixed_sizes), memory_begin)
{
}

Expand Down Expand Up @@ -284,23 +283,17 @@ class AllFixedSizeElementLocator : public BaseAllFixedSizeElementLocator
trivially_copy_into(*this, new_memory_begin);
}

constexpr auto calculate_new_memory_size(std::size_t max_element_count, std::size_t varying_size_bytes,
const FixedSizesArray&) noexcept
constexpr std::size_t calculate_new_memory_size(std::size_t max_element_count, std::size_t varying_size_bytes,
const FixedSizesArray&) noexcept
{
return varying_size_bytes + stride_ * max_element_count;
}

private:
void trivially_copy_into(const AllFixedSizeElementLocator& old_locator, std::byte* new_memory_begin) noexcept
{
const auto new_start = AllFixedSizeElementLocator::calculate_element_start(new_memory_begin);
std::memcpy(new_start, old_locator.start_, old_locator.element_count_ * old_locator.stride_);
start_ = new_start;
}

static constexpr auto calculate_element_start(std::byte* memory_begin) noexcept
{
return detail::align<ElementTraits::template ParameterTraitsAt<0>::ALIGNMENT>(memory_begin);
std::memcpy(new_memory_begin, old_locator.start_, old_locator.element_count_ * old_locator.stride_);
start_ = new_memory_begin;
}
};

Expand Down
128 changes: 79 additions & 49 deletions src/cntgs/detail/elementTraits.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,6 @@

namespace cntgs::detail
{
template <std::size_t Alignment>
constexpr auto alignment_offset(std::size_t position) noexcept
{
return detail::align<Alignment>(position) - position;
}

template <bool UseMove, class Type, class Source, class Target>
constexpr void construct_one_if_non_trivial([[maybe_unused]] Source&& source, [[maybe_unused]] const Target& target)
{
Expand All @@ -44,6 +38,12 @@ constexpr void construct_one_if_non_trivial([[maybe_unused]] Source&& source, [[
}
}

struct ElementSize
{
std::size_t size;
std::size_t stride;
};

template <class, class...>
class ElementTraits;

Expand All @@ -52,14 +52,21 @@ class ElementTraits<std::index_sequence<I...>, Parameter...>
{
private:
using ListTraits = detail::ParameterListTraits<Parameter...>;

public:
template <std::size_t K>
using ParameterTraitsAt = typename ListTraits::template ParameterTraitsAt<K>;

private:
using FixedSizesArray = typename ListTraits::FixedSizesArray;
using ContiguousPointer = typename detail::ContiguousVectorTraits<Parameter...>::PointerType;
using ContiguousReference = typename detail::ContiguousVectorTraits<Parameter...>::ReferenceType;
using FixedSizeGetter = detail::FixedSizeGetter<Parameter...>;

static constexpr auto ALIGNMENT_OF_FIRST_PARAMETER = ListTraits::template ParameterTraitsAt<0>::ALIGNMENT;
static constexpr auto SKIP = std::numeric_limits<std::size_t>::max();
static constexpr auto MANUAL = SKIP - 1;
static constexpr std::size_t LARGEST_LEADING_ALIGNMENT_UNTIL_VARYING_SIZE =
ListTraits::LARGEST_LEADING_ALIGNMENT_UNTIL_VARYING_SIZE;
static constexpr std::size_t SKIP = std::numeric_limits<std::size_t>::max();
static constexpr std::size_t MANUAL = SKIP - 1;

template <template <class> class Predicate>
static constexpr auto calculate_consecutive_indices() noexcept
Expand Down Expand Up @@ -106,87 +113,110 @@ class ElementTraits<std::index_sequence<I...>, Parameter...>
ParameterTraitsAt<K>::data_begin(cntgs::get<K>(rhs))};
}

static constexpr ElementSize calculate_element_size(const FixedSizesArray& fixed_sizes) noexcept
{
std::size_t size{};
std::size_t offset{};
std::size_t padding{};
(
[&]
{
const auto [next_offset, next_size, next_padding] =
detail::ParameterTraits<Parameter>::template aligned_size_in_memory<
ListTraits::template previous_alignment<I>(), ListTraits::template next_alignment<I>()>(
offset, FixedSizeGetter::template get<Parameter, I>(fixed_sizes));
size += next_size;
offset = next_offset;
if constexpr (I == sizeof...(Parameter) - 1)
{
padding = next_padding;
}
}(),
...);
return {size, size + padding};
}

template <class ParameterT, bool IgnoreAliasing, std::size_t K, class Args>
static std::byte* store_one(std::byte* address, std::size_t fixed_size, Args&& args) noexcept
{
static constexpr auto PREVIOUS_ALIGNMENT = ListTraits::template previous_alignment<K>();
return detail::ParameterTraits<ParameterT>::template store<PREVIOUS_ALIGNMENT, IgnoreAliasing>(
std::forward<Args>(args), address, fixed_size);
}

template <bool IgnoreAliasing, class... Args>
static std::byte* emplace_at(std::byte* CNTGS_RESTRICT address, const FixedSizesArray& fixed_sizes, Args&&... args)
static std::byte* emplace_at(std::byte* address, const FixedSizesArray& fixed_sizes, Args&&... args)
{
((address = detail::ParameterTraits<Parameter>::template store<true, IgnoreAliasing>(
std::forward<Args>(args), address, FixedSizeGetter::template get<Parameter, I>(fixed_sizes))),
((address = store_one<Parameter, IgnoreAliasing, I>(
address, FixedSizeGetter::template get<Parameter, I>(fixed_sizes), std::forward<Args>(args))),
...);
return address;
}

template <class Type, std::size_t K, class FixedSizeGetterType, class FixedSizesType>
template <class ParameterT, std::size_t K, class FixedSizeGetterType, class FixedSizesType>
static auto load_one(std::byte* CNTGS_RESTRICT address, const FixedSizesType& fixed_sizes) noexcept
{
static constexpr auto NEEDS_ALIGNMENT = K != 0;
static constexpr auto IS_SIZE_PROVIDED = FixedSizeGetterType::template CAN_PROVIDE_SIZE<Type>;
return detail::ParameterTraits<Type>::template load<NEEDS_ALIGNMENT, IS_SIZE_PROVIDED>(
address, FixedSizeGetterType::template get<Type, K>(fixed_sizes));
static constexpr auto PREVIOUS_ALIGNMENT = ListTraits::template previous_alignment<K>();
static constexpr auto IS_SIZE_PROVIDED = FixedSizeGetterType::template CAN_PROVIDE_SIZE<ParameterT>;
return detail::ParameterTraits<ParameterT>::template load<PREVIOUS_ALIGNMENT, IS_SIZE_PROVIDED>(
address, FixedSizeGetterType::template get<ParameterT, K>(fixed_sizes));
}

public:
template <std::size_t K>
using ParameterTraitsAt = typename ListTraits::template ParameterTraitsAt<K>;

using StorageElementType = detail::Aligned<ALIGNMENT_OF_FIRST_PARAMETER>;
using StorageElementType = detail::Aligned<LARGEST_LEADING_ALIGNMENT_UNTIL_VARYING_SIZE>;

template <class StorageType, class Allocator>
static constexpr auto allocate_memory(std::size_t size_in_bytes, const Allocator& allocator)
static constexpr StorageType allocate_memory(std::size_t size_in_bytes, const Allocator& allocator)
{
const auto remainder = size_in_bytes % ElementTraits::ALIGNMENT_OF_FIRST_PARAMETER;
auto count = size_in_bytes / ElementTraits::ALIGNMENT_OF_FIRST_PARAMETER;
const auto remainder = size_in_bytes % LARGEST_LEADING_ALIGNMENT_UNTIL_VARYING_SIZE;
auto count = size_in_bytes / LARGEST_LEADING_ALIGNMENT_UNTIL_VARYING_SIZE;
count += remainder == 0 ? 0 : 1;
return StorageType(count, allocator);
}

static constexpr std::byte* align_for_first_parameter(std::byte* address) noexcept
{
return detail::align<ALIGNMENT_OF_FIRST_PARAMETER>(address);
return detail::align_if<(ListTraits::template trailing_alignment<(sizeof...(I) - 1)>()) <
LARGEST_LEADING_ALIGNMENT_UNTIL_VARYING_SIZE,
LARGEST_LEADING_ALIGNMENT_UNTIL_VARYING_SIZE>(address);
}

template <class... Args>
CNTGS_RESTRICT_RETURN static std::byte* emplace_at(std::byte* CNTGS_RESTRICT address,
const FixedSizesArray& fixed_sizes, Args&&... args)
{
return ElementTraits::template emplace_at<true>(address, fixed_sizes, std::forward<Args>(args)...);
return emplace_at<true>(address, fixed_sizes, std::forward<Args>(args)...);
}

template <class... Args>
static std::byte* emplace_at_aliased(std::byte* CNTGS_RESTRICT address, const FixedSizesArray& fixed_sizes,
Args&&... args)
static std::byte* emplace_at_aliased(std::byte* address, const FixedSizesArray& fixed_sizes, Args&&... args)
{
return ElementTraits::template emplace_at<false>(address, fixed_sizes, std::forward<Args>(args)...);
return emplace_at<false>(address, fixed_sizes, std::forward<Args>(args)...);
}

template <class FixedSizeGetterType = ElementTraits::FixedSizeGetter,
class FixedSizesType = ElementTraits::FixedSizesArray>
static auto load_element_at(std::byte* CNTGS_RESTRICT address, const FixedSizesType& fixed_sizes) noexcept
static ContiguousPointer load_element_at(std::byte* CNTGS_RESTRICT address,
const FixedSizesType& fixed_sizes) noexcept
{
ContiguousPointer result;
((std::tie(std::get<I>(result), address) =
ElementTraits::template load_one<Parameter, I, FixedSizeGetterType>(address, fixed_sizes)),
((std::tie(std::get<I>(result), address) = load_one<Parameter, I, FixedSizeGetterType>(address, fixed_sizes)),
...);
return result;
}

static constexpr auto calculate_element_size(const FixedSizesArray& fixed_sizes) noexcept
static constexpr std::size_t calculate_element_stride(const FixedSizesArray& fixed_sizes) noexcept
{
std::size_t result{};
if constexpr (ListTraits::IS_FIXED_SIZE_OR_PLAIN)
{
((result += detail::ParameterTraits<Parameter>::guaranteed_size_in_memory(
FixedSizeGetter::template get<Parameter, I>(fixed_sizes)) +
detail::alignment_offset<detail::ParameterTraits<Parameter>::ALIGNMENT>(result)),
...);
}
else
{
((result += detail::ParameterTraits<Parameter>::aligned_size_in_memory(
FixedSizeGetter::template get<Parameter, I>(fixed_sizes)) +
detail::alignment_offset<detail::ParameterTraits<Parameter>::ALIGNMENT>(result)),
...);
}
return result + detail::alignment_offset<ALIGNMENT_OF_FIRST_PARAMETER>(result);
return calculate_element_size(fixed_sizes).stride;
}

static constexpr std::size_t calculate_needed_memory_size(std::size_t max_element_count,
std::size_t varying_size_bytes,
const FixedSizesArray& fixed_sizes) noexcept
{
const auto [element_size, element_stride] = calculate_element_size(fixed_sizes);
const auto padding = max_element_count == 0 ? 0 : (element_stride - element_size);
return varying_size_bytes + element_stride * max_element_count - padding;
}

template <bool UseMove, bool IsConst>
Expand Down
40 changes: 34 additions & 6 deletions src/cntgs/detail/memory.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,16 +10,13 @@
#include "cntgs/detail/iterator.hpp"
#include "cntgs/detail/range.hpp"
#include "cntgs/detail/typeTraits.hpp"
#include "cntgs/detail/utility.hpp"
#include "cntgs/span.hpp"

#include <algorithm>
#include <cstddef>
#include <cstring>
#include <iterator>
#include <limits>
#include <memory>
#include <version>

namespace cntgs::detail
{
Expand Down Expand Up @@ -78,7 +75,7 @@ constexpr auto uninitialized_copy_n(SourceIterator&& source, DifferenceType coun
}

template <bool IgnoreAliasing, class TargetType, class Range>
auto uninitialized_range_construct(Range&& CNTGS_RESTRICT range, TargetType* CNTGS_RESTRICT address)
auto uninitialized_range_construct(Range&& range, TargetType* address)
{
using RangeValueType = typename std::iterator_traits<decltype(std::begin(range))>::value_type;
if constexpr (IgnoreAliasing && detail::HAS_DATA_AND_SIZE<std::decay_t<Range>> &&
Expand All @@ -100,14 +97,14 @@ auto uninitialized_range_construct(Range&& CNTGS_RESTRICT range, TargetType* CNT
}

template <bool IgnoreAliasing, class TargetType, class Range>
auto uninitialized_construct(Range&& CNTGS_RESTRICT range, TargetType* CNTGS_RESTRICT address,
auto uninitialized_construct(Range&& range, TargetType* address,
std::size_t) -> std::enable_if_t<detail::IS_RANGE<Range>, std::byte*>
{
return detail::uninitialized_range_construct<IgnoreAliasing>(std::forward<Range>(range), address);
}

template <bool IgnoreAliasing, class TargetType, class Iterator>
auto uninitialized_construct(const Iterator& CNTGS_RESTRICT iterator, TargetType* CNTGS_RESTRICT address,
auto uninitialized_construct(const Iterator& iterator, TargetType* address,
std::size_t size) -> std::enable_if_t<!detail::IS_RANGE<Iterator>, std::byte*>
{
using IteratorValueType = typename std::iterator_traits<Iterator>::value_type;
Expand Down Expand Up @@ -147,6 +144,14 @@ template <std::size_t Alignment, class T>
}
#endif

[[nodiscard]] inline bool is_aligned(void* ptr, size_t alignment) noexcept
{
void* void_ptr = ptr;
auto size = std::numeric_limits<size_t>::max();
std::align(alignment, 0, void_ptr, size);
return void_ptr == ptr;
}

[[nodiscard]] constexpr auto align(std::uintptr_t position, std::size_t alignment) noexcept
{
return (position - 1u + alignment) & (alignment * std::numeric_limits<std::size_t>::max());
Expand Down Expand Up @@ -189,6 +194,29 @@ template <bool NeedsAlignment, std::size_t Alignment, class T>
}
return detail::assume_aligned<Alignment>(ptr);
}

template <bool NeedsAlignment, std::size_t Alignment>
[[nodiscard]] constexpr auto align_if(std::uintptr_t position) noexcept
{
if constexpr (NeedsAlignment && Alignment > 1)
{
position = detail::align<Alignment>(position);
}
return position;
}

template <class T>
[[nodiscard]] constexpr T extract_lowest_set_bit(T value) noexcept
{
return value & (~value + T{1});
}

[[nodiscard]] constexpr std::size_t trailing_alignment(std::size_t byte_size, std::size_t alignment) noexcept
{
return (std::min)(detail::extract_lowest_set_bit(byte_size), alignment);
}

inline constexpr auto SIZE_T_TRAILING_ALIGNMENT = detail::trailing_alignment(sizeof(std::size_t), alignof(std::size_t));
} // namespace cntgs::detail

#endif // CNTGS_DETAIL_MEMORY_HPP
Loading

0 comments on commit 23c1df0

Please sign in to comment.