diff --git a/include/hwmalloc/detail/fixed_size_heap.hpp b/include/hwmalloc/detail/fixed_size_heap.hpp index e58292b..435984b 100644 --- a/include/hwmalloc/detail/fixed_size_heap.hpp +++ b/include/hwmalloc/detail/fixed_size_heap.hpp @@ -37,7 +37,8 @@ class fixed_size_heap public: fixed_size_heap(Context* context, std::size_t block_size, std::size_t segment_size, - bool never_free, std::size_t num_reserve_segments) + bool never_free, std::size_t num_reserve_segments, + const typename pool_type::segment_alloc_cb_type& segment_alloc_cb = nullptr) : m_context(context) , m_block_size(block_size) , m_segment_size(segment_size) @@ -52,12 +53,13 @@ class fixed_size_heap for (auto [n, i] : numa().local_nodes()) { m_pools[i] = std::make_unique(m_context, m_block_size, m_segment_size, n, - m_never_free, m_num_reserve_segments); + m_never_free, m_num_reserve_segments, segment_alloc_cb); #if HWMALLOC_ENABLE_DEVICE for (unsigned int j = 0; j < m_num_devices; ++j) { m_device_pools[i * m_num_devices + j] = std::make_unique(m_context, - m_block_size, m_segment_size, n, (int)j, m_never_free, m_num_reserve_segments); + m_block_size, m_segment_size, n, (int)j, m_never_free, m_num_reserve_segments, + segment_alloc_cb); } #endif } diff --git a/include/hwmalloc/detail/pool.hpp b/include/hwmalloc/detail/pool.hpp index 85fda28..920eab1 100644 --- a/include/hwmalloc/detail/pool.hpp +++ b/include/hwmalloc/detail/pool.hpp @@ -14,6 +14,7 @@ #include #include #include +#include namespace hwmalloc { @@ -27,6 +28,7 @@ class pool using block_type = typename segment_type::block; using stack_type = boost::lockfree::stack; using segment_map = std::unordered_map>; + using segment_alloc_cb_type = std::function; private: static std::size_t num_pages(std::size_t segment_size) noexcept @@ -48,20 +50,25 @@ class pool } private: - Context* m_context; - std::size_t m_block_size; - std::size_t m_segment_size; - std::size_t m_numa_node; - bool m_never_free; - std::size_t m_num_reserve_segments; - stack_type m_free_stack; - segment_map m_segments; - std::mutex m_mutex; - int m_device_id = 0; - bool m_allocate_on_device = false; + Context* m_context; + std::size_t m_block_size; + std::size_t m_segment_size; + std::size_t m_numa_node; + bool m_never_free; + std::size_t m_num_reserve_segments; + stack_type m_free_stack; + segment_map m_segments; + std::mutex m_mutex; + int m_device_id = 0; + bool m_allocate_on_device = false; + // Callback executed when a segment is allocated. Can be used to track the number of actual + // memory allocations. + const segment_alloc_cb_type& m_segment_alloc_cb; void add_segment() { + if (m_segment_alloc_cb) + m_segment_alloc_cb(); auto a = check_allocation(numa().allocate(num_pages(m_segment_size), m_numa_node), m_numa_node); #if HWMALLOC_ENABLE_DEVICE @@ -90,7 +97,8 @@ class pool public: pool(Context* context, std::size_t block_size, std::size_t segment_size, std::size_t numa_node, - bool never_free, std::size_t num_reserve_segments) + bool never_free, std::size_t num_reserve_segments, + const segment_alloc_cb_type& segment_alloc_cb) : m_context{context} , m_block_size{block_size} , m_segment_size{segment_size} @@ -98,13 +106,15 @@ class pool , m_never_free{never_free} , m_num_reserve_segments{std::max(num_reserve_segments, 1ul)} , m_free_stack(segment_size / block_size) + , m_segment_alloc_cb(segment_alloc_cb) { } #if HWMALLOC_ENABLE_DEVICE pool(Context* context, std::size_t block_size, std::size_t segment_size, std::size_t numa_node, - int device_id, bool never_free, std::size_t num_reserve_segments) - : pool(context, block_size, segment_size, numa_node, never_free, num_reserve_segments) + int device_id, bool never_free, std::size_t num_reserve_segments, + const segment_alloc_cb_type& segment_alloc_cb) + : pool(context, block_size, segment_size, numa_node, never_free, num_reserve_segments, segment_alloc_cb) { m_device_id = device_id; m_allocate_on_device = true; diff --git a/include/hwmalloc/heap.hpp b/include/hwmalloc/heap.hpp index 8d400b8..b426050 100644 --- a/include/hwmalloc/heap.hpp +++ b/include/hwmalloc/heap.hpp @@ -1,7 +1,7 @@ /* * ghex-org * - * Copyright (c) 2014-2023, ETH Zurich + * Copyright (c) 2014-2025, ETH Zurich * All rights reserved. * * Please, refer to the LICENSE file in the root directory. @@ -14,9 +14,11 @@ #include #include #include +#include #include #include #include +#include namespace hwmalloc { @@ -56,6 +58,9 @@ class heap template using unique_ptr = unique_ptr; + // Note: sizes below are defaults and can be changed through heap_config and + // environment variables. + // // There are 5 size classes that the heap uses. For each size class it relies on a // fixed_size_heap. The size classes are: // - tiny: heaps with linearly increasing block sizes, each heap backed by 16KiB segments @@ -98,78 +103,71 @@ class heap // : : m_huge_heaps: map private: - static constexpr std::size_t log2_c(std::size_t n) noexcept + static std::size_t tiny_bucket_index(std::size_t n, std::size_t tiny_increment, + std::size_t tiny_increment_shift) noexcept { - return ((n < 2) ? 1 : 1 + log2_c(n >> 1)); + return ((n + tiny_increment - 1) >> tiny_increment_shift) - 1; } - static const std::size_t s_tiny_limit = (1u << 7); // 128 - static const std::size_t s_small_limit = (1u << 10); // 1024 - static const std::size_t s_large_limit = (1u << 16); // 65536 - - static const std::size_t s_bucket_shift = log2_c(s_tiny_limit) - 1; - - static const std::size_t s_tiny_segment = 0x04000; // 16KiB - static const std::size_t s_small_segment = 0x08000; // 32KiB - static const std::size_t s_large_segment = 0x10000; // 64KiB - - static const std::size_t s_tiny_increment_shift = 3; - static const std::size_t s_tiny_increment = (1u << s_tiny_increment_shift); // = 8 - - static const std::size_t s_num_tiny_heaps = s_tiny_limit / s_tiny_increment; - static const std::size_t s_num_small_heaps = log2_c(s_small_limit) - log2_c(s_tiny_limit); - static const std::size_t s_num_large_heaps = log2_c(s_large_limit) - log2_c(s_small_limit); - - static std::size_t tiny_bucket_index(std::size_t n) noexcept + static std::size_t bucket_index(std::size_t n, std::size_t bucket_shift) noexcept { - return ((n + s_tiny_increment - 1) >> s_tiny_increment_shift) - 1; - } - - static std::size_t bucket_index(std::size_t n) noexcept - { - return log2_c((n - 1) >> s_bucket_shift) - 1; - } - - static constexpr std::size_t round_to_pow_of_2(std::size_t n) noexcept - { - return 1u << log2_c(n - 1); + return detail::log2_c((n - 1) >> bucket_shift) - 1; } private: + heap_config m_config; Context* m_context; std::size_t m_max_size; - bool m_never_free; - std::size_t m_num_reserve_segments; heap_vector m_tiny_heaps; heap_vector m_heaps; heap_map m_huge_heaps; std::mutex m_mutex; + std::size_t m_num_huge_alloc; + bool m_num_huge_alloc_did_warn = false; + typename fixed_size_heap_type::pool_type::segment_alloc_cb_type + m_huge_segment_alloc_cb; public: - heap(Context* context, bool never_free = false, std::size_t num_reserve_segments = 1) - : m_context{context} - , m_max_size(std::max(round_to_pow_of_2(s_large_limit * 2), s_large_limit)) - , m_never_free{never_free} - , m_num_reserve_segments{num_reserve_segments} - , m_tiny_heaps(s_tiny_limit / s_tiny_increment) - , m_heaps(bucket_index(m_max_size) + 1) + heap(Context* context, heap_config const& config = get_default_heap_config()) + : m_config{config} + , m_context{context} + , m_max_size( + std::max(detail::round_to_pow_of_2(m_config.m_large_limit * 2), m_config.m_large_limit)) + , m_tiny_heaps(m_config.m_tiny_limit / m_config.m_tiny_increment) + , m_heaps(bucket_index(m_max_size, m_config.m_bucket_shift) + 1) { for (std::size_t i = 0; i < m_tiny_heaps.size(); ++i) m_tiny_heaps[i] = std::make_unique(m_context, - s_tiny_increment * (i + 1), s_tiny_segment, m_never_free, m_num_reserve_segments); + m_config.m_tiny_increment * (i + 1), m_config.m_tiny_segment_size, + m_config.m_never_free, m_config.m_num_reserve_segments); - for (std::size_t i = 0; i < s_num_small_heaps; ++i) + for (std::size_t i = 0; i < m_config.m_num_small_heaps; ++i) m_heaps[i] = std::make_unique(m_context, - (s_tiny_limit << (i + 1)), s_small_segment, m_never_free, m_num_reserve_segments); + (m_config.m_tiny_limit << (i + 1)), m_config.m_small_segment_size, + m_config.m_never_free, m_config.m_num_reserve_segments); + + for (std::size_t i = 0; i < m_config.m_num_large_heaps; ++i) + m_heaps[i + m_config.m_num_small_heaps] = std::make_unique( + m_context, (m_config.m_small_limit << (i + 1)), m_config.m_large_segment_size, + m_config.m_never_free, m_config.m_num_reserve_segments); - for (std::size_t i = 0; i < s_num_large_heaps; ++i) - m_heaps[i + s_num_small_heaps] = std::make_unique(m_context, - (s_small_limit << (i + 1)), s_large_segment, m_never_free, m_num_reserve_segments); + for (std::size_t i = 0; + i < m_heaps.size() - (m_config.m_num_small_heaps + m_config.m_num_large_heaps); ++i) + m_heaps[i + m_config.m_num_small_heaps + m_config.m_num_large_heaps] = + std::make_unique(m_context, + (m_config.m_large_limit << (i + 1)), (m_config.m_large_limit << (i + 1)), + m_config.m_never_free, m_config.m_num_reserve_segments); - for (std::size_t i = 0; i < m_heaps.size() - (s_num_small_heaps + s_num_large_heaps); ++i) - m_heaps[i + s_num_small_heaps + s_num_large_heaps] = - std::make_unique(m_context, (s_large_limit << (i + 1)), - (s_large_limit << (i + 1)), m_never_free, m_num_reserve_segments); + if (m_config.m_num_huge_alloc_warn_threshold > 0) + m_huge_segment_alloc_cb = [this] () { + m_num_huge_alloc += 1; + if (!m_num_huge_alloc_did_warn && m_num_huge_alloc >= m_config.m_num_huge_alloc_warn_threshold) { + m_num_huge_alloc_did_warn = true; + std::cerr + << "WARNING: huge allocation count exceeds HWMALLOC_NUM_HUGE_ALLOC_WARN_THERESHOLD=" + << m_config.m_num_huge_alloc_warn_threshold << std::endl; + } + }; } heap(heap const&) = delete; @@ -194,20 +192,23 @@ class heap pointer allocate(std::size_t size, std::size_t numa_node) { - if (size <= s_tiny_limit) - return {m_tiny_heaps[tiny_bucket_index(size)]->allocate(numa_node)}; + if (size <= m_config.m_tiny_limit) + return {m_tiny_heaps[tiny_bucket_index(size, m_config.m_tiny_increment, + m_config.m_tiny_increment_shift)] + ->allocate(numa_node)}; else if (size <= m_max_size) - return {m_heaps[bucket_index(size)]->allocate(numa_node)}; + return {m_heaps[bucket_index(size, m_config.m_bucket_shift)]->allocate(numa_node)}; else { fixed_size_heap_type* h; { std::lock_guard lock(m_mutex); - const auto s = round_to_pow_of_2(size); + const auto s = detail::round_to_pow_of_2(size); auto& u_ptr = m_huge_heaps[s]; - if (!u_ptr) - u_ptr = std::make_unique(m_context, s, s, m_never_free, - m_num_reserve_segments); + if (!u_ptr) { + u_ptr = std::make_unique(m_context, s, s, + m_config.m_never_free, m_config.m_num_reserve_segments, m_huge_segment_alloc_cb); + } h = u_ptr.get(); } return {h->allocate(numa_node)}; @@ -223,20 +224,27 @@ class heap #if HWMALLOC_ENABLE_DEVICE pointer allocate(std::size_t size, std::size_t numa_node, int device_id) { - if (size <= s_tiny_limit) - return {m_tiny_heaps[tiny_bucket_index(size)]->allocate(numa_node, device_id)}; + if (size <= m_config.m_tiny_limit) + return {m_tiny_heaps[tiny_bucket_index(size, m_config.m_tiny_increment, + m_config.m_tiny_increment_shift)] + ->allocate(numa_node, device_id)}; else if (size <= m_max_size) - return {m_heaps[bucket_index(size)]->allocate(numa_node, device_id)}; + return {m_heaps[bucket_index(size, m_config.m_bucket_shift)]->allocate(numa_node, + device_id)}; else { fixed_size_heap_type* h; { std::lock_guard lock(m_mutex); - const auto s = round_to_pow_of_2(size); + const auto s = detail::round_to_pow_of_2(size); auto& u_ptr = m_huge_heaps[s]; - if (!u_ptr) - u_ptr = std::make_unique(m_context, s, s, m_never_free, - m_num_reserve_segments); + if (!u_ptr) { + typename fixed_size_heap_type::pool_type::segment_alloc_cb_type& segment_alloc_cb; + if (m_config.m_num_huge_alloc_warn_threshold > 0) + segment_alloc_cb = std::bind(&heap::huge_alloc_cb, this); + u_ptr = std::make_unique(m_context, s, s, + m_config.m_never_free, m_config.m_num_reserve_segments, m_huge_segment_alloc_cb); + } h = u_ptr.get(); } return {h->allocate(numa_node, device_id)}; @@ -293,29 +301,4 @@ class heap } }; -template -const std::size_t heap::s_tiny_limit; -template -const std::size_t heap::s_small_limit; -template -const std::size_t heap::s_large_limit; -template -const std::size_t heap::s_bucket_shift; -template -const std::size_t heap::s_tiny_segment; -template -const std::size_t heap::s_small_segment; -template -const std::size_t heap::s_large_segment; -template -const std::size_t heap::s_tiny_increment_shift; -template -const std::size_t heap::s_tiny_increment; -template -const std::size_t heap::s_num_tiny_heaps; -template -const std::size_t heap::s_num_small_heaps; -template -const std::size_t heap::s_num_large_heaps; - } // namespace hwmalloc diff --git a/include/hwmalloc/heap_config.hpp b/include/hwmalloc/heap_config.hpp new file mode 100644 index 0000000..2cf906a --- /dev/null +++ b/include/hwmalloc/heap_config.hpp @@ -0,0 +1,55 @@ +/* + * ghex-org + * + * Copyright (c) 2014-2025, ETH Zurich + * All rights reserved. + * + * Please, refer to the LICENSE file in the root directory. + * SPDX-License-Identifier: BSD-3-Clause + */ +#pragma once + +#include + +namespace hwmalloc +{ +namespace detail +{ +inline constexpr std::size_t +log2_c(std::size_t n) noexcept +{ + return ((n < 2) ? 1 : 1 + log2_c(n >> 1)); +} + +inline constexpr std::size_t +round_to_pow_of_2(std::size_t n) noexcept +{ + return 1u << log2_c(n - 1); +} +} // namespace detail + +struct heap_config +{ + bool m_never_free; + std::size_t m_num_reserve_segments; + std::size_t m_tiny_limit; + std::size_t m_small_limit; + std::size_t m_large_limit; + std::size_t m_bucket_shift = detail::log2_c(m_tiny_limit) - 1; + std::size_t m_tiny_segment_size; + std::size_t m_small_segment_size; + std::size_t m_large_segment_size; + std::size_t m_num_huge_alloc_warn_threshold; + static constexpr std::size_t m_tiny_increment_shift = 3; + static constexpr std::size_t m_tiny_increment = (1u << m_tiny_increment_shift); + std::size_t m_num_tiny_heaps = m_tiny_limit / m_tiny_increment; + std::size_t m_num_small_heaps = detail::log2_c(m_small_limit) - detail::log2_c(m_tiny_limit); + std::size_t m_num_large_heaps = detail::log2_c(m_large_limit) - detail::log2_c(m_small_limit); + + heap_config(bool never_free, std::size_t num_reserve_segments, std::size_t tiny_limit, + std::size_t small_limit, std::size_t large_limit, std::size_t tiny_segment_size, + std::size_t small_segment_size, std::size_t large_segment_size, std::size_t num_huge_alloc_warn_threshold); +}; + +heap_config const& get_default_heap_config(); +} // namespace hwmalloc diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 66e14a6..042a508 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,3 +1,5 @@ +target_sources(hwmalloc PRIVATE heap_config.cpp) + if (NUMA_LIBRARY) target_sources(hwmalloc PRIVATE numa.cpp) else() diff --git a/src/heap_config.cpp b/src/heap_config.cpp new file mode 100644 index 0000000..45954f1 --- /dev/null +++ b/src/heap_config.cpp @@ -0,0 +1,132 @@ +/* + * ghex-org + * + * Copyright (c) 2014-2025, ETH Zurich + * All rights reserved. + * + * Please, refer to the LICENSE file in the root directory. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include +#include + +#ifdef HWMALLOC_ENABLE_LOGGING +#include + +#include +#endif + +#include +#include +#include + +namespace hwmalloc +{ +namespace detail +{ +static bool +get_env_bool(const char* name, bool default_value) noexcept +{ + const char* env_value = std::getenv(name); + if (env_value) + { + try + { + return std::stoul(env_value) != 0; + } + catch (...) + { +#ifdef HWMALLOC_ENABLE_LOGGING + HWMALLOC_LOG("failed to parse boolean configuration option", name, "=", env_value, + "(expected 0 or 1), using default =", default_value); +#endif + return default_value; + } + } + + return default_value; +} + +static std::size_t +get_env_size_t(const char* name, std::size_t default_value) noexcept +{ + const char* env_value = std::getenv(name); + if (env_value) + { + try + { + return std::stoul(env_value); + } + catch (...) + { +#ifdef HWMALLOC_ENABLE_LOGGING + HWMALLOC_LOG("failed to parse configuration option", name, "=", env_value, + ", using default =", default_value); +#endif + return default_value; + } + } + + return default_value; +} +} // namespace detail + +heap_config::heap_config(bool never_free, std::size_t num_reserve_segments, std::size_t tiny_limit, + std::size_t small_limit, std::size_t large_limit, std::size_t tiny_segment_size, + std::size_t small_segment_size, std::size_t large_segment_size, std::size_t num_huge_alloc_warn_threshold) +: m_never_free{never_free} +, m_num_reserve_segments{num_reserve_segments} +, m_tiny_limit{detail::round_to_pow_of_2(tiny_limit)} +, m_small_limit{detail::round_to_pow_of_2(small_limit)} +, m_large_limit{detail::round_to_pow_of_2(large_limit)} +, m_tiny_segment_size{detail::round_to_pow_of_2(tiny_segment_size)} +, m_small_segment_size{detail::round_to_pow_of_2(small_segment_size)} +, m_large_segment_size{detail::round_to_pow_of_2(large_segment_size)} +, m_num_huge_alloc_warn_threshold{num_huge_alloc_warn_threshold} +{ + // Validate that tiny_limit < small_limit < large_limit + if (!(m_tiny_limit < m_small_limit && m_small_limit < m_large_limit)) + { + std::ostringstream os; + os << "Invalid heap size configuration: HWMALLOC_TINY_LIMIT < HWMALLOC_SMALL_LIMIT < HWMALLOC_LARGE_LIMIT must hold. "; + os << "Got HWMALLOC_TINY_LIMIT=" << m_tiny_limit + << ", HWMALLOC_SMALL_LIMIT=" << m_small_limit + << ", HWMALLOC_LARGE_LIMIT=" << m_large_limit << "."; + throw std::runtime_error(os.str()); + } + + // Validate that limits are at most segment sizes + if (!(tiny_limit <= tiny_segment_size && small_limit <= small_segment_size && + large_limit <= large_segment_size)) + { + std::ostringstream os; + os << "Invalid heap size configuration: Limits must be at most segment sizes. "; + os << "Got HWMALLOC_TINY_LIMIT=" << m_tiny_limit + << ", HWMALLOC_TINY_SEGMENT_SIZE=" << m_tiny_segment_size + << ", HWMALLOC_SMALL_LIMIT=" << m_small_limit + << ", HWMALLOC_SMALL_SEGMENT_SIZE=" << m_small_segment_size + << ", HWMALLOC_LARGE_LIMIT=" << m_large_limit + << ", HWMALLOC_LARGE_SEGMENT_SIZE=" << m_large_segment_size << "."; + throw std::runtime_error(os.str()); + } +} + +heap_config const& +get_default_heap_config() +{ + static heap_config config{ + detail::get_env_bool("HWMALLOC_NEVER_FREE", false), + detail::get_env_size_t("HWMALLOC_NUM_RESERVE_SEGMENTS", 16u), + detail::get_env_size_t("HWMALLOC_TINY_LIMIT", (1u << 7)), // 128B + detail::get_env_size_t("HWMALLOC_SMALL_LIMIT", (1u << 12)), // 4KiB + detail::get_env_size_t("HWMALLOC_LARGE_LIMIT", (1u << 21)), // 2MiB + detail::get_env_size_t("HWMALLOC_TINY_SEGMENT_SIZE", (1u << 16)), // 64KiB + detail::get_env_size_t("HWMALLOC_SMALL_SEGMENT_SIZE", (1u << 16)), // 64KiB + detail::get_env_size_t("HWMALLOC_LARGE_SEGMENT_SIZE", (1u << 21)), // 2MiB + detail::get_env_size_t("HWMALLOC_NUM_HUGE_ALLOC_WARN_THERESHOLD", 10) + }; + + return config; +} +} // namespace hwmalloc diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index f339a3c..859b97c 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -33,6 +33,9 @@ if (NUMA_LIBRARY AND NOT HWMALLOC_DISABLE_NUMA_TEST) endif() reg_test(test_ptr) reg_test(test_segment) +reg_test(test_heap_config) +reg_test(test_heap_config_default) +reg_test(test_heap_config_invalid) if (NUMA_LIBRARY) find_package(OpenMP REQUIRED) diff --git a/test/test_heap_config.cpp b/test/test_heap_config.cpp new file mode 100644 index 0000000..851592c --- /dev/null +++ b/test/test_heap_config.cpp @@ -0,0 +1,286 @@ +/* + * ghex-org + * + * Copyright (c) 2014-2025, ETH Zurich + * All rights reserved. + * + * Please, refer to the LICENSE file in the root directory. + * SPDX-License-Identifier: BSD-3-Clause + */ +#include + +#include + +#include + +// These should change accordingly if the defaults in heap_config.cpp change. +// They are defined here for a little bit of convenience in the tests below. +inline constexpr bool never_free_default = false; +inline constexpr std::size_t num_reserve_segments_default = 16u; +inline constexpr std::size_t tiny_limit_default = 128u; +inline constexpr std::size_t small_limit_default = 4096u; +inline constexpr std::size_t large_limit_default = 2097152u; +inline constexpr std::size_t bucket_shift_default = 7u; +inline constexpr std::size_t tiny_segment_size_default = 65536u; +inline constexpr std::size_t small_segment_size_default = 65536u; +inline constexpr std::size_t large_segment_size_default = 2097152u; +inline constexpr std::size_t tiny_increment_shift_default = 3u; +inline constexpr std::size_t tiny_increment_default = 8u; +inline constexpr std::size_t num_tiny_heaps_default = 16u; +inline constexpr std::size_t num_small_heaps_default = 5u; +inline constexpr std::size_t num_large_heaps_default = 9u; + +// Check that the default configuration values are as expected. +TEST(heap_config, defaults) +{ + hwmalloc::heap_config config = hwmalloc::get_default_heap_config(); + + EXPECT_EQ(config.m_never_free, never_free_default); + EXPECT_EQ(config.m_num_reserve_segments, num_reserve_segments_default); + EXPECT_EQ(config.m_tiny_limit, tiny_limit_default); + EXPECT_EQ(config.m_small_limit, small_limit_default); + EXPECT_EQ(config.m_large_limit, large_limit_default); + EXPECT_EQ(config.m_bucket_shift, bucket_shift_default); + EXPECT_EQ(config.m_tiny_segment_size, tiny_segment_size_default); + EXPECT_EQ(config.m_small_segment_size, small_segment_size_default); + EXPECT_EQ(config.m_large_segment_size, large_segment_size_default); + EXPECT_EQ(config.m_tiny_increment_shift, tiny_increment_shift_default); + EXPECT_EQ(config.m_tiny_increment, tiny_increment_default); + EXPECT_EQ(config.m_num_tiny_heaps, num_tiny_heaps_default); + EXPECT_EQ(config.m_num_small_heaps, num_small_heaps_default); + EXPECT_EQ(config.m_num_large_heaps, num_large_heaps_default); +} + +// We try changing one parameter a time compared to the defaults and check that +// the derived values change accordingly. +TEST(heap_config, never_free) +{ + hwmalloc::heap_config config{true, num_reserve_segments_default, tiny_limit_default, + small_limit_default, large_limit_default, tiny_segment_size_default, + small_segment_size_default, large_segment_size_default}; + + EXPECT_EQ(config.m_never_free, true); // This changes to the literal value we set + EXPECT_EQ(config.m_num_reserve_segments, num_reserve_segments_default); + EXPECT_EQ(config.m_tiny_limit, tiny_limit_default); + EXPECT_EQ(config.m_small_limit, small_limit_default); + EXPECT_EQ(config.m_large_limit, large_limit_default); + EXPECT_EQ(config.m_bucket_shift, bucket_shift_default); + EXPECT_EQ(config.m_tiny_segment_size, tiny_segment_size_default); + EXPECT_EQ(config.m_small_segment_size, small_segment_size_default); + EXPECT_EQ(config.m_large_segment_size, large_segment_size_default); + EXPECT_EQ(config.m_tiny_increment_shift, tiny_increment_shift_default); + EXPECT_EQ(config.m_tiny_increment, tiny_increment_default); + EXPECT_EQ(config.m_num_tiny_heaps, num_tiny_heaps_default); + EXPECT_EQ(config.m_num_small_heaps, num_small_heaps_default); + EXPECT_EQ(config.m_num_large_heaps, num_large_heaps_default); +} + +TEST(heap_config, num_reserve_segments) +{ + hwmalloc::heap_config config{never_free_default, 7, tiny_limit_default, small_limit_default, + large_limit_default, tiny_segment_size_default, small_segment_size_default, + large_segment_size_default}; + + EXPECT_EQ(config.m_never_free, never_free_default); + EXPECT_EQ(config.m_num_reserve_segments, 7u); // This changes to the literal value we set + EXPECT_EQ(config.m_tiny_limit, tiny_limit_default); + EXPECT_EQ(config.m_small_limit, small_limit_default); + EXPECT_EQ(config.m_large_limit, large_limit_default); + EXPECT_EQ(config.m_bucket_shift, bucket_shift_default); + EXPECT_EQ(config.m_tiny_segment_size, tiny_segment_size_default); + EXPECT_EQ(config.m_small_segment_size, small_segment_size_default); + EXPECT_EQ(config.m_large_segment_size, large_segment_size_default); + EXPECT_EQ(config.m_tiny_increment_shift, tiny_increment_shift_default); + EXPECT_EQ(config.m_tiny_increment, tiny_increment_default); + EXPECT_EQ(config.m_num_tiny_heaps, num_tiny_heaps_default); + EXPECT_EQ(config.m_num_small_heaps, num_small_heaps_default); + EXPECT_EQ(config.m_num_large_heaps, num_large_heaps_default); +} + +TEST(heap_config, tiny_limit) +{ + hwmalloc::heap_config config{never_free_default, num_reserve_segments_default, 256u, + small_limit_default, large_limit_default, tiny_segment_size_default, + small_segment_size_default, large_segment_size_default}; + + EXPECT_EQ(config.m_never_free, never_free_default); + EXPECT_EQ(config.m_num_reserve_segments, num_reserve_segments_default); + EXPECT_EQ(config.m_tiny_limit, 256u); // This changes to the literal value we set + EXPECT_EQ(config.m_small_limit, small_limit_default); + EXPECT_EQ(config.m_large_limit, large_limit_default); + EXPECT_EQ(config.m_bucket_shift, 8u); // This changes based on tiny_limit + EXPECT_EQ(config.m_tiny_segment_size, tiny_segment_size_default); + EXPECT_EQ(config.m_small_segment_size, small_segment_size_default); + EXPECT_EQ(config.m_large_segment_size, large_segment_size_default); + EXPECT_EQ(config.m_tiny_increment_shift, tiny_increment_shift_default); + EXPECT_EQ(config.m_tiny_increment, tiny_increment_default); + EXPECT_EQ(config.m_num_tiny_heaps, 32u); // This number doubles... + EXPECT_EQ(config.m_num_small_heaps, 4u); // While this is decreased by one + EXPECT_EQ(config.m_num_large_heaps, num_large_heaps_default); +} + +TEST(heap_config, tiny_segment) +{ + hwmalloc::heap_config config{never_free_default, num_reserve_segments_default, + tiny_limit_default, small_limit_default, large_limit_default, 8192u, + small_segment_size_default, large_segment_size_default}; + + EXPECT_EQ(config.m_never_free, never_free_default); + EXPECT_EQ(config.m_num_reserve_segments, num_reserve_segments_default); + EXPECT_EQ(config.m_tiny_limit, tiny_limit_default); + EXPECT_EQ(config.m_small_limit, small_limit_default); + EXPECT_EQ(config.m_large_limit, large_limit_default); + EXPECT_EQ(config.m_bucket_shift, bucket_shift_default); + EXPECT_EQ(config.m_tiny_segment_size, 8192u); // This changes to the literal value we set + EXPECT_EQ(config.m_small_segment_size, small_segment_size_default); + EXPECT_EQ(config.m_large_segment_size, large_segment_size_default); + EXPECT_EQ(config.m_tiny_increment_shift, tiny_increment_shift_default); + EXPECT_EQ(config.m_tiny_increment, tiny_increment_default); + EXPECT_EQ(config.m_num_tiny_heaps, num_tiny_heaps_default); + EXPECT_EQ(config.m_num_small_heaps, num_small_heaps_default); + EXPECT_EQ(config.m_num_large_heaps, num_large_heaps_default); +} + +TEST(heap_config, small_limit) +{ + hwmalloc::heap_config config{never_free_default, num_reserve_segments_default, + tiny_limit_default, 8192u, large_limit_default, tiny_segment_size_default, + small_segment_size_default, large_segment_size_default}; + + EXPECT_EQ(config.m_never_free, never_free_default); + EXPECT_EQ(config.m_num_reserve_segments, num_reserve_segments_default); + EXPECT_EQ(config.m_tiny_limit, tiny_limit_default); + EXPECT_EQ(config.m_small_limit, 8192u); // This changes to the literal value we set + EXPECT_EQ(config.m_large_limit, large_limit_default); + EXPECT_EQ(config.m_bucket_shift, bucket_shift_default); + EXPECT_EQ(config.m_tiny_segment_size, tiny_segment_size_default); + EXPECT_EQ(config.m_small_segment_size, small_segment_size_default); + EXPECT_EQ(config.m_large_segment_size, large_segment_size_default); + EXPECT_EQ(config.m_tiny_increment_shift, tiny_increment_shift_default); + EXPECT_EQ(config.m_tiny_increment, tiny_increment_default); + EXPECT_EQ(config.m_num_tiny_heaps, num_tiny_heaps_default); + EXPECT_EQ(config.m_num_small_heaps, 6u); // This is log2(8192)-log2(128) = 6... + EXPECT_EQ(config.m_num_large_heaps, 8u); // While this is decreased accordingly +} + +TEST(heap_config, small_segment) +{ + hwmalloc::heap_config config{never_free_default, num_reserve_segments_default, + tiny_limit_default, small_limit_default, large_limit_default, tiny_segment_size_default, + 16384u, large_segment_size_default}; + + EXPECT_EQ(config.m_never_free, never_free_default); + EXPECT_EQ(config.m_num_reserve_segments, num_reserve_segments_default); + EXPECT_EQ(config.m_tiny_limit, tiny_limit_default); + EXPECT_EQ(config.m_small_limit, small_limit_default); + EXPECT_EQ(config.m_large_limit, large_limit_default); + EXPECT_EQ(config.m_bucket_shift, bucket_shift_default); + EXPECT_EQ(config.m_tiny_segment_size, tiny_segment_size_default); + EXPECT_EQ(config.m_small_segment_size, 16384u); // This changes to the literal value we set + EXPECT_EQ(config.m_large_segment_size, large_segment_size_default); + EXPECT_EQ(config.m_tiny_increment_shift, tiny_increment_shift_default); + EXPECT_EQ(config.m_tiny_increment, tiny_increment_default); + EXPECT_EQ(config.m_num_tiny_heaps, num_tiny_heaps_default); + EXPECT_EQ(config.m_num_small_heaps, num_small_heaps_default); + EXPECT_EQ(config.m_num_large_heaps, num_large_heaps_default); +} + +TEST(heap_config, large_limit) +{ + hwmalloc::heap_config config{never_free_default, num_reserve_segments_default, + tiny_limit_default, small_limit_default, 32768u, tiny_segment_size_default, + small_segment_size_default, large_segment_size_default}; + + EXPECT_EQ(config.m_never_free, never_free_default); + EXPECT_EQ(config.m_num_reserve_segments, num_reserve_segments_default); + EXPECT_EQ(config.m_tiny_limit, tiny_limit_default); + EXPECT_EQ(config.m_small_limit, small_limit_default); + EXPECT_EQ(config.m_large_limit, 32768u); // This changes to the literal value we set + EXPECT_EQ(config.m_bucket_shift, bucket_shift_default); + EXPECT_EQ(config.m_tiny_segment_size, tiny_segment_size_default); + EXPECT_EQ(config.m_small_segment_size, small_segment_size_default); + EXPECT_EQ(config.m_large_segment_size, large_segment_size_default); + EXPECT_EQ(config.m_tiny_increment_shift, tiny_increment_shift_default); + EXPECT_EQ(config.m_tiny_increment, tiny_increment_default); + EXPECT_EQ(config.m_num_tiny_heaps, num_tiny_heaps_default); + EXPECT_EQ(config.m_num_small_heaps, num_small_heaps_default); + EXPECT_EQ(config.m_num_large_heaps, 3u); // This is decreased accordingly +} + +TEST(heap_config, large_segment) +{ + hwmalloc::heap_config config{never_free_default, num_reserve_segments_default, + tiny_limit_default, small_limit_default, large_limit_default, tiny_segment_size_default, + small_segment_size_default, 4194304u}; + + EXPECT_EQ(config.m_never_free, never_free_default); + EXPECT_EQ(config.m_num_reserve_segments, num_reserve_segments_default); + EXPECT_EQ(config.m_tiny_limit, tiny_limit_default); + EXPECT_EQ(config.m_small_limit, small_limit_default); + EXPECT_EQ(config.m_large_limit, large_limit_default); + EXPECT_EQ(config.m_bucket_shift, bucket_shift_default); + EXPECT_EQ(config.m_tiny_segment_size, tiny_segment_size_default); + EXPECT_EQ(config.m_small_segment_size, small_segment_size_default); + EXPECT_EQ(config.m_large_segment_size, 4194304u); // This changes to the literal value we set + EXPECT_EQ(config.m_tiny_increment_shift, tiny_increment_shift_default); + EXPECT_EQ(config.m_tiny_increment, tiny_increment_default); + EXPECT_EQ(config.m_num_tiny_heaps, num_tiny_heaps_default); + EXPECT_EQ(config.m_num_small_heaps, num_small_heaps_default); + EXPECT_EQ(config.m_num_large_heaps, num_large_heaps_default); +} + +// Check that setting non-power-of-two values results in rounding to power of two +TEST(heap_config, power_of_two) +{ + hwmalloc::heap_config config{never_free_default, 17u, 200u, 500u, 20000u, 1000u, 3000u, + 100000u}; + + EXPECT_EQ(config.m_num_reserve_segments, 17u); // Not rounded up + EXPECT_EQ(config.m_tiny_limit, 256u); // Rounded up from 200 + EXPECT_EQ(config.m_small_limit, 512u); // Rounded up from 500 + EXPECT_EQ(config.m_large_limit, 32768u); // Rounded up from 20000 + EXPECT_EQ(config.m_tiny_segment_size, 1024u); // Rounded up from 1000 + EXPECT_EQ(config.m_small_segment_size, 4096u); // Rounded up from 3000 + EXPECT_EQ(config.m_large_segment_size, 131072u); // Rounded up from 100000 +} + +// Check that setting invalid values results in exceptions +TEST(heap_config, validate_tiny_small_limit) +{ + EXPECT_THROW((hwmalloc::heap_config{never_free_default, num_reserve_segments_default, 2048u, + 1024u, large_limit_default, tiny_segment_size_default, + small_segment_size_default, large_segment_size_default}), + std::runtime_error); +} + +TEST(heap_config, validate_small_large_limit) +{ + EXPECT_THROW((hwmalloc::heap_config{never_free_default, num_reserve_segments_default, + tiny_limit_default, 131072u, 65536u, tiny_segment_size_default, + small_segment_size_default, large_segment_size_default}), + std::runtime_error); +} + +TEST(heap_config, validate_tiny_limit_segment_size) +{ + EXPECT_THROW((hwmalloc::heap_config{never_free_default, num_reserve_segments_default, 2048u, + small_limit_default, large_limit_default, 1024u, small_segment_size_default, + large_segment_size_default}), + std::runtime_error); +} + +TEST(heap_config, validate_small_limit_segment_size) +{ + EXPECT_THROW((hwmalloc::heap_config{never_free_default, num_reserve_segments_default, + tiny_limit_default, 131072u, large_limit_default, tiny_segment_size_default, + 65536u, large_segment_size_default}), + std::runtime_error); +} + +TEST(heap_config, validate_large_limit_segment_size) +{ + EXPECT_THROW((hwmalloc::heap_config{never_free_default, num_reserve_segments_default, + tiny_limit_default, small_limit_default, 262144u, tiny_segment_size_default, + small_segment_size_default, 131072u}), + std::runtime_error); +} diff --git a/test/test_heap_config_default.cpp b/test/test_heap_config_default.cpp new file mode 100644 index 0000000..885be0d --- /dev/null +++ b/test/test_heap_config_default.cpp @@ -0,0 +1,45 @@ +/* + * ghex-org + * + * Copyright (c) 2014-2025, ETH Zurich + * All rights reserved. + * + * Please, refer to the LICENSE file in the root directory. + * SPDX-License-Identifier: BSD-3-Clause + */ +#include + +#include + +#include + +// Check that we can change the default configuration values through environment +// variables. +TEST(config, defaults) +{ + ::setenv("HWMALLOC_NEVER_FREE", "1", 1); + ::setenv("HWMALLOC_NUM_RESERVE_SEGMENTS", "19", 1); + ::setenv("HWMALLOC_TINY_LIMIT", "512", 1); + ::setenv("HWMALLOC_SMALL_LIMIT", "2000", 1); + ::setenv("HWMALLOC_LARGE_LIMIT", "131072", 1); + ::setenv("HWMALLOC_TINY_SEGMENT_SIZE", "16384", 1); + ::setenv("HWMALLOC_SMALL_SEGMENT_SIZE", "32768", 1); + ::setenv("HWMALLOC_LARGE_SEGMENT_SIZE", "262144", 1); + + hwmalloc::heap_config config = hwmalloc::get_default_heap_config(); + + EXPECT_EQ(config.m_never_free, true); + EXPECT_EQ(config.m_num_reserve_segments, 19u); + EXPECT_EQ(config.m_tiny_limit, 512u); + EXPECT_EQ(config.m_small_limit, 2048); + EXPECT_EQ(config.m_large_limit, 131072u); + EXPECT_EQ(config.m_bucket_shift, 9u); + EXPECT_EQ(config.m_tiny_segment_size, 16384u); + EXPECT_EQ(config.m_small_segment_size, 32768u); + EXPECT_EQ(config.m_large_segment_size, 262144u); + EXPECT_EQ(config.m_tiny_increment_shift, 3u); + EXPECT_EQ(config.m_tiny_increment, 8u); + EXPECT_EQ(config.m_num_tiny_heaps, 64u); + EXPECT_EQ(config.m_num_small_heaps, 2u); + EXPECT_EQ(config.m_num_large_heaps, 6u); +} diff --git a/test/test_heap_config_invalid.cpp b/test/test_heap_config_invalid.cpp new file mode 100644 index 0000000..dd30e09 --- /dev/null +++ b/test/test_heap_config_invalid.cpp @@ -0,0 +1,64 @@ +/* + * ghex-org + * + * Copyright (c) 2014-2025, ETH Zurich + * All rights reserved. + * + * Please, refer to the LICENSE file in the root directory. + * SPDX-License-Identifier: BSD-3-Clause + */ +#include + +#include + +#include + +// These should change accordingly if the defaults in heap_config.cpp change. +// They are defined here for a little bit of convenience in the tests below. +inline constexpr bool never_free_default = false; +inline constexpr std::size_t num_reserve_segments_default = 16u; +inline constexpr std::size_t tiny_limit_default = 128u; +inline constexpr std::size_t small_limit_default = 4096u; +inline constexpr std::size_t large_limit_default = 2097152u; +inline constexpr std::size_t bucket_shift_default = 7u; +inline constexpr std::size_t tiny_segment_size_default = 65536u; +inline constexpr std::size_t small_segment_size_default = 65536u; +inline constexpr std::size_t large_segment_size_default = 2097152u; +inline constexpr std::size_t tiny_increment_shift_default = 3u; +inline constexpr std::size_t tiny_increment_default = 8u; +inline constexpr std::size_t num_tiny_heaps_default = 16u; +inline constexpr std::size_t num_small_heaps_default = 5u; +inline constexpr std::size_t num_large_heaps_default = 9u; + +// Test that config falls back to defaults if environment variables are given +// non-numeric values. +// +// A warning is printed if logging is enabled. +TEST(config, defaults) +{ + ::setenv("HWMALLOC_NEVER_FREE", "foo", 1); + ::setenv("HWMALLOC_NUM_RESERVE_SEGMENTS", "9", 1); + ::setenv("HWMALLOC_TINY_LIMIT", "abcd", 1); + ::setenv("HWMALLOC_SMALL_LIMIT", "8192", 1); + ::setenv("HWMALLOC_LARGE_LIMIT", "foo", 1); + ::setenv("HWMALLOC_TINY_SEGMENT_SIZE", "16384", 1); + ::setenv("HWMALLOC_SMALL_SEGMENT_SIZE", "bar", 1); + ::setenv("HWMALLOC_LARGE_SEGMENT_SIZE", "4194304", 1); + + hwmalloc::heap_config config = hwmalloc::get_default_heap_config(); + + EXPECT_EQ(config.m_never_free, false); + EXPECT_EQ(config.m_num_reserve_segments, 9u); + EXPECT_EQ(config.m_tiny_limit, tiny_limit_default); + EXPECT_EQ(config.m_small_limit, 8192); + EXPECT_EQ(config.m_large_limit, large_limit_default); + EXPECT_EQ(config.m_bucket_shift, bucket_shift_default); + EXPECT_EQ(config.m_tiny_segment_size, 16384u); + EXPECT_EQ(config.m_small_segment_size, small_segment_size_default); + EXPECT_EQ(config.m_large_segment_size, 4194304u); + EXPECT_EQ(config.m_tiny_increment_shift, tiny_increment_shift_default); + EXPECT_EQ(config.m_tiny_increment, tiny_increment_default); + EXPECT_EQ(config.m_num_tiny_heaps, num_tiny_heaps_default); + EXPECT_EQ(config.m_num_small_heaps, 6u); + EXPECT_EQ(config.m_num_large_heaps, 8u); +} diff --git a/test/test_omp.cpp b/test/test_omp.cpp index c4b38ca..fb59fc3 100644 --- a/test/test_omp.cpp +++ b/test/test_omp.cpp @@ -73,8 +73,13 @@ register_memory(context&, void* ptr, std::size_t size) TEST(spread, neverfree) { using heap_t = hwmalloc::heap; + using heap_config_t = hwmalloc::heap_config; context c; - heap_t h(&c, 1024, true); + + heap_config_t hc = hwmalloc::get_default_heap_config(); + hc.m_never_free = true; + hc.m_num_reserve_segments = 1024; + heap_t h(&c, hc); n_registrations = 0; #pragma omp parallel @@ -144,8 +149,13 @@ TEST(spread, neverfree) TEST(close, neverfree) { using heap_t = hwmalloc::heap; + using heap_config_t = hwmalloc::heap_config; context c; - heap_t h(&c, 1024, true); + + heap_config_t hc = hwmalloc::get_default_heap_config(); + hc.m_never_free = true; + hc.m_num_reserve_segments = 1024; + heap_t h(&c, hc); n_registrations = 0; #pragma omp parallel @@ -212,8 +222,13 @@ TEST(close, neverfree) TEST(spread, free) { using heap_t = hwmalloc::heap; + using heap_config_t = hwmalloc::heap_config; context c; - heap_t h(&c); + + heap_config_t hc = hwmalloc::get_default_heap_config(); + hc.m_never_free = false; + hc.m_num_reserve_segments = 1; + heap_t h(&c, hc); n_registrations = 0; #pragma omp parallel @@ -275,8 +290,7 @@ TEST(spread, free) #pragma omp master { printf("SPREAD, free :: n_registrations %d\n", n_registrations); - EXPECT_TRUE( - n_registrations == (thr_per_node * NITER * nnodes * (NBUFFERS - 1) + nnodes)); + EXPECT_EQ(n_registrations, (thr_per_node * NITER * nnodes * (NBUFFERS - 1) + nnodes)); } } } @@ -284,8 +298,13 @@ TEST(spread, free) TEST(close, free) { using heap_t = hwmalloc::heap; + using heap_config_t = hwmalloc::heap_config; context c; - heap_t h(&c); + + heap_config_t hc = hwmalloc::get_default_heap_config(); + hc.m_never_free = false; + hc.m_num_reserve_segments = 1; + heap_t h(&c, hc); n_registrations = 0; #pragma omp parallel @@ -343,7 +362,7 @@ TEST(close, free) #pragma omp master { printf("CLOSE, free :: n_registrations %d\n", n_registrations); - EXPECT_TRUE(n_registrations == (nthr * NITER * (NBUFFERS - 1) + nused_nodes)); + EXPECT_EQ(n_registrations, (nthr * NITER * (NBUFFERS - 1) + nused_nodes)); } } }