diff --git a/CMakeLists.txt b/CMakeLists.txt index 392b53e80..69864469c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -379,7 +379,7 @@ endif() function(add_warning_flags name) target_compile_options(${name} PRIVATE $<$:/Zi /W4 /WX /wd4127 /wd4324 /wd4201> - $<$,$>>:-fno-exceptions -fno-rtti -Wall -Wextra -Werror -Wundef> + $<$,$>>:-fno-rtti -Wall -Wextra -Werror -Wundef> $<$:-Wsign-conversion -Wconversion>) target_link_options(${name} PRIVATE $<$:-Wl,--no-undefined> @@ -503,29 +503,29 @@ if(NOT SNMALLOC_HEADER_ONLY_LIBRARY) message(WARNING "Compiler does not support `-march=native` required by SNMALLOC_OPTIMISE_FOR_CURRENT_MACHINE") set(SNMALLOC_OPTIMISE_FOR_CURRENT_MACHINE FALSE) endif() -endif() + endif() - function(add_shim name type) - add_library(${name} ${type} ${ARGN}) + function(compile name TYPE ${ARGN}) + add_library(${name} ${TYPE} ${ARGN}) target_link_libraries(${name} snmalloc) - set_target_properties(${name} PROPERTIES CXX_VISIBILITY_PRESET hidden INTERPROCEDURAL_OPTIMIZATION ${SNMALLOC_COMPILER_SUPPORT_IPO}) target_compile_definitions(${name} PRIVATE "SNMALLOC_USE_${SNMALLOC_CLEANUP}") - if(MSVC) - target_compile_definitions(${name} PRIVATE -D_HAS_EXCEPTIONS=0) - endif() - add_warning_flags(${name}) if(NOT MSVC) target_compile_definitions(${name} PRIVATE "SNMALLOC_EXPORT=__attribute__((visibility(\"default\")))") - target_compile_options(${name} PRIVATE - -fomit-frame-pointer -ffunction-sections) + set_target_properties(${name} PROPERTIES CXX_VISIBILITY_PRESET hidden INTERPROCEDURAL_OPTIMIZATION ${SNMALLOC_COMPILER_SUPPORT_IPO}) + # Make debugging harder, and code faster. + target_compile_options(${name} PRIVATE -fomit-frame-pointer) + + set_property(TARGET ${name} PROPERTY POSITION_INDEPENDENT_CODE ON) + # Check for prefetch write support. check_cxx_compiler_flag("-Werror -Wextra -Wall -mprfchw" SUPPORT_PREFETCH_WRITE) if (SUPPORT_PREFETCH_WRITE) target_compile_options(${name} PRIVATE -mprfchw) endif() + # Static TLS model is unsupported on Haiku. if (SNMALLOC_STATIC_MODE_TLS) target_compile_options(${name} PRIVATE -ftls-model=initial-exec) @@ -536,18 +536,8 @@ endif() target_compile_options(${name} PRIVATE -march=native) endif() - # Ensure that we do not link against C++ stdlib when compiling shims. - # If the compiler supports excluding the C++ stdlib implementation, use - # it. Otherwise, fall back to linking the library as if it were C, which - # has roughly the same effect. - if (NOT ${SNMALLOC_CLEANUP} STREQUAL CXX11_DESTRUCTORS) - if (SNMALLOC_LINKER_SUPPORT_NOSTDLIBXX) - target_link_options(${name} PRIVATE -nostdlib++) - else() - set_target_properties(${name} PROPERTIES LINKER_LANGUAGE C) - endif() - endif() # Remove all the duplicate new/malloc and free/delete definitions + target_compile_options(${name} PRIVATE -ffunction-sections) target_link_options(${name} PRIVATE $<$:$<$:-Wl,--icf=all> -fuse-ld=lld>) endif() @@ -556,40 +546,59 @@ endif() target_compile_definitions(${name} PRIVATE SNMALLOC_PAGEID=$,true,false>) - install(TARGETS ${name} EXPORT snmallocConfig) + if(MSVC) + else() + # We can't do --nostdlib++ as we are using exceptions, but we + # can make it a static dependency, which we aren't really using + # as the interesting stuff for exceptions is in libgcc_s + target_link_options(${name} PRIVATE -static-libstdc++) + endif() + if (SNMALLOC_RUST_LIBC_API) + target_compile_definitions(${name} PRIVATE SNMALLOC_RUST_LIBC_API) + endif() + install(TARGETS ${name} EXPORT snmallocConfig) endfunction() - set(SHIM_FILES src/snmalloc/override/malloc.cc src/snmalloc/override/new.cc) - set(SHIM_FILES_MEMCPY src/snmalloc/override/memcpy.cc) + # Various files for overriding libc/rust behaviours. + set(MALLOC src/snmalloc/override/malloc.cc) + set(NEW src/snmalloc/override/new.cc) + set(MEMCPY src/snmalloc/override/memcpy.cc) + set(RUST src/snmalloc/override/rust.cc) - add_shim(snmalloc-new-override STATIC src/snmalloc/override/new.cc) + set(ALLOC ${MALLOC} ${NEW}) + set(ALL ${ALLOC} ${MEMCPY}) if (SNMALLOC_STATIC_LIBRARY) - add_shim(snmallocshim-static STATIC ${SHIM_FILES}) + compile(snmallocshim-static STATIC ${ALLOC}) target_compile_definitions(snmallocshim-static PRIVATE SNMALLOC_STATIC_LIBRARY_PREFIX=${SNMALLOC_STATIC_LIBRARY_PREFIX}) endif () + compile(snmalloc-new-override STATIC ${NEW}) + if(NOT WIN32) - add_shim(snmallocshim SHARED ${SHIM_FILES}) + compile(snmallocshim SHARED ${ALLOC}) + if (SNMALLOC_MEMCPY_OVERRIDE) - add_shim(snmallocshim-checks-memcpy-only SHARED ${SHIM_FILES} ${SHIM_FILES_MEMCPY}) - add_shim(snmallocshim-checks SHARED ${SHIM_FILES} ${SHIM_FILES_MEMCPY}) + compile(snmallocshim-checks-memcpy-only SHARED ${ALL}) + compile(snmallocshim-checks SHARED ${ALL}) else() - add_shim(snmallocshim-checks SHARED ${SHIM_FILES}) + compile(snmallocshim-checks SHARED ${ALLOC}) endif() target_compile_definitions(snmallocshim-checks PRIVATE SNMALLOC_CHECK_CLIENT) + + compile(snmalloc-minimal SHARED ${MALLOC}) + target_compile_options(snmalloc-minimal PRIVATE -fno-exceptions) + if (SNMALLOC_LINKER_SUPPORT_NOSTDLIBXX AND NOT ${SNMALLOC_CLEANUP} STREQUAL CXX11_DESTRUCTORS) + target_compile_options(snmalloc-minimal PRIVATE -fno-exceptions -nostdlib++) + endif () endif() if(SNMALLOC_RUST_SUPPORT) - add_shim(snmallocshim-rust STATIC src/snmalloc/override/rust.cc) - add_shim(snmallocshim-checks-rust STATIC src/snmalloc/override/rust.cc) + compile(snmallocshim-rust STATIC ${RUST}) + compile(snmallocshim-checks-rust STATIC ${RUST}) target_compile_definitions(snmallocshim-checks-rust PRIVATE SNMALLOC_CHECK_CLIENT) - if (SNMALLOC_RUST_LIBC_API) - target_compile_definitions(snmallocshim-rust PRIVATE SNMALLOC_RUST_LIBC_API) - target_compile_definitions(snmallocshim-checks-rust PRIVATE SNMALLOC_RUST_LIBC_API) - endif() endif() if (SNMALLOC_BUILD_TESTING) @@ -625,7 +634,7 @@ endif() foreach (MITIGATION ${MITIGATIONS}) set(DEFINES "SNMALLOC_CHECK_CLIENT_MITIGATIONS=${MITIGATION}") - add_shim(snmallocshim-${MITIGATION} SHARED ${SHIM_FILES}) + compile(snmallocshim-${MITIGATION} SHARED ${ALLOC}) target_compile_definitions(snmallocshim-${MITIGATION} PRIVATE ${DEFINES}) if (SNMALLOC_BUILD_TESTING) make_tests(${MITIGATION} ${DEFINES}) @@ -640,7 +649,7 @@ endif() set(MITIGATIONSET "${MITIGATIONSET}+${MITIGATION}") message(STATUS "MITIGATIONSET: ${COUNT} -> ${MITIGATIONSET}") set(DEFINES "-DSNMALLOC_CHECK_CLIENT_MITIGATIONS=${MITIGATIONSET}") - add_shim(snmallocshim-${MITIGATIONNAME} SHARED ${SHIM_FILES}) + compile(snmallocshim-${MITIGATIONNAME} SHARED ${ALLOC}) target_compile_definitions(snmallocshim-${MITIGATIONNAME} PRIVATE ${DEFINES}) if (SNMALLOC_BUILD_TESTING) make_tests(${MITIGATIONNAME} ${DEFINES}) diff --git a/src/snmalloc/mem/corealloc.h b/src/snmalloc/mem/corealloc.h index 1a8ce98aa..d83ab9dc7 100644 --- a/src/snmalloc/mem/corealloc.h +++ b/src/snmalloc/mem/corealloc.h @@ -50,7 +50,7 @@ namespace snmalloc return p; } - static void* failure(size_t size) + static void* failure(size_t size) noexcept { UNUSED(size); // If we are here, then the allocation failed. @@ -401,9 +401,9 @@ namespace snmalloc * the stack as often closing over the arguments would cause less good * codegen. */ - template + template SNMALLOC_FAST_PATH decltype(auto) - handle_message_queue(Action action, Args... args) + handle_message_queue(Action action, Args... args) noexcept(noexc) { // Inline the empty check, but not necessarily the full queue handling. if (SNMALLOC_LIKELY(!has_messages())) @@ -411,15 +411,15 @@ namespace snmalloc return action(args...); } - return handle_message_queue_slow(action, args...); + return handle_message_queue_slow(action, args...); } /** * Process remote frees into this allocator. */ - template + template SNMALLOC_SLOW_PATH decltype(auto) - handle_message_queue_slow(Action action, Args... args) + handle_message_queue_slow(Action action, Args... args) noexcept(noexc) { bool need_post = false; size_t bytes_freed = 0; @@ -602,7 +602,8 @@ namespace snmalloc * required. It is defaulted to do nothing. */ template - SNMALLOC_FAST_PATH ALLOCATOR void* alloc(size_t size) + SNMALLOC_FAST_PATH ALLOCATOR void* + alloc(size_t size) noexcept(noexcept(Conts::failure(0))) { // Perform the - 1 on size, so that zero wraps around and ends up on // slow path. @@ -621,7 +622,8 @@ namespace snmalloc * Fast allocation for small objects. */ template - SNMALLOC_FAST_PATH void* small_alloc(size_t size) + SNMALLOC_FAST_PATH void* + small_alloc(size_t size) noexcept(noexcept(Conts::failure(0))) { auto domesticate = [this](freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { @@ -637,7 +639,7 @@ namespace snmalloc return finish_alloc(p, size); } - return handle_message_queue( + return handle_message_queue( []( Allocator* alloc, smallsizeclass_t sizeclass, @@ -661,8 +663,8 @@ namespace snmalloc * register. */ template - static SNMALLOC_SLOW_PATH void* - alloc_not_small(size_t size, Allocator* self) + static SNMALLOC_SLOW_PATH void* alloc_not_small( + size_t size, Allocator* self) noexcept(noexcept(Conts::failure(0))) { if (size == 0) { @@ -672,7 +674,7 @@ namespace snmalloc return self->small_alloc(1); } - return self->handle_message_queue( + return self->template handle_message_queue( [](Allocator* self, size_t size) SNMALLOC_FAST_PATH_LAMBDA { return CheckInit::check_init( [self, size]() SNMALLOC_FAST_PATH_LAMBDA { @@ -739,7 +741,9 @@ namespace snmalloc template SNMALLOC_FAST_PATH void* small_refill( - smallsizeclass_t sizeclass, freelist::Iter<>& fast_free_list, size_t size) + smallsizeclass_t sizeclass, + freelist::Iter<>& fast_free_list, + size_t size) noexcept(noexcept(Conts::failure(0))) { void* result = Config::SecondaryAllocator::allocate( [size]() -> stl::Pair { @@ -809,7 +813,9 @@ namespace snmalloc template SNMALLOC_SLOW_PATH void* small_refill_slow( - smallsizeclass_t sizeclass, freelist::Iter<>& fast_free_list, size_t size) + smallsizeclass_t sizeclass, + freelist::Iter<>& fast_free_list, + size_t size) noexcept(noexcept(Conts::failure(0))) { return CheckInit::check_init( [this, size, sizeclass, &fast_free_list]() SNMALLOC_FAST_PATH_LAMBDA { @@ -994,7 +1000,7 @@ namespace snmalloc * deallocation to the other allocators. ***************************************************************************/ template - SNMALLOC_FAST_PATH void dealloc(void* p_raw) + SNMALLOC_FAST_PATH void dealloc(void* p_raw) noexcept { #ifdef __CHERI_PURE_CAPABILITY__ /* @@ -1043,7 +1049,7 @@ namespace snmalloc SNMALLOC_FAST_PATH void dealloc_local_object( CapPtr p, - const typename Config::PagemapEntry& entry) + const typename Config::PagemapEntry& entry) noexcept { auto meta = entry.get_slab_metadata(); @@ -1078,7 +1084,7 @@ namespace snmalloc SNMALLOC_SLOW_PATH void dealloc_local_object_slow( capptr::Alloc p, const PagemapEntry& entry, - BackendSlabMetadata* meta) + BackendSlabMetadata* meta) noexcept { // TODO: Handle message queue on this path? @@ -1273,8 +1279,8 @@ namespace snmalloc } template - SNMALLOC_FAST_PATH void - dealloc_remote(const PagemapEntry& entry, capptr::Alloc p_tame) + SNMALLOC_FAST_PATH void dealloc_remote( + const PagemapEntry& entry, capptr::Alloc p_tame) noexcept { if (SNMALLOC_LIKELY(entry.is_owned())) { @@ -1327,8 +1333,8 @@ namespace snmalloc * as we might acquire the originating allocator. */ template - SNMALLOC_SLOW_PATH void - dealloc_remote_slow(const PagemapEntry& entry, capptr::Alloc p) + SNMALLOC_SLOW_PATH void dealloc_remote_slow( + const PagemapEntry& entry, capptr::Alloc p) noexcept { CheckInit::check_init( [this, &entry, p]() SNMALLOC_FAST_PATH_LAMBDA { @@ -1344,7 +1350,7 @@ namespace snmalloc post(); }, - [](Allocator* a, void* p) { + [](Allocator* a, void* p) SNMALLOC_FAST_PATH_LAMBDA { // Recheck what kind of dealloc we should do in case the allocator // we get from lazy_init is the originating allocator. a->dealloc(p); // TODO don't double count statistics @@ -1385,7 +1391,7 @@ namespace snmalloc // Process incoming message queue // Loop as normally only processes a batch while (has_messages()) - handle_message_queue([]() {}); + handle_message_queue([]() {}); } auto& key = freelist::Object::key_root; diff --git a/src/snmalloc/mem/ticker.h b/src/snmalloc/mem/ticker.h index c48ff193e..33e3819fb 100644 --- a/src/snmalloc/mem/ticker.h +++ b/src/snmalloc/mem/ticker.h @@ -41,7 +41,7 @@ namespace snmalloc * how many calls for the next time we hit the slow path. */ template - SNMALLOC_SLOW_PATH T check_tick_slow(T p = nullptr) + SNMALLOC_SLOW_PATH T check_tick_slow(T p = nullptr) noexcept { uint64_t now_ms = PAL::time_in_ms(); diff --git a/src/snmalloc/override/new.cc b/src/snmalloc/override/new.cc index 3f0efc183..7a97c6651 100644 --- a/src/snmalloc/override/new.cc +++ b/src/snmalloc/override/new.cc @@ -20,24 +20,100 @@ // only define these if we are not using the vendored STL #ifndef SNMALLOC_USE_SELF_VENDORED_STL + +namespace snmalloc +{ + namespace handler + { + class Base + { + public: + static void* + success(void* p, size_t size, bool secondary_allocator = false) + { + UNUSED(secondary_allocator, size); + SNMALLOC_ASSERT(p != nullptr); + + SNMALLOC_ASSERT( + secondary_allocator || + is_start_of_object(size_to_sizeclass_full(size), address_cast(p))); + + return p; + } + }; + + class Throw : public Base + { + public: + static void* failure(size_t size) + { + // Throw std::bad_alloc on failure. + auto new_handler = std::get_new_handler(); + if (new_handler != nullptr) + { + // Call the new handler, which may throw an exception. + new_handler(); + // Retry now new_handler has been called. + // I dislike the unbounded retrying here, but that seems to be what + // other implementations do. + return alloc(size); + } + + // Throw std::bad_alloc on failure. + throw std::bad_alloc(); + } + }; + + class NoThrow : public Base + { + public: + static void* failure(size_t size) noexcept + { + auto new_handler = std::get_new_handler(); + if (new_handler != nullptr) + { + try + { + // Call the new handler, which may throw an exception. + new_handler(); + } + catch (...) + { + // If the new handler throws, we just return nullptr. + return nullptr; + } + // Retry now new_handler has been called. + return alloc(size); + } + // If we are here, then the allocation failed. + // Set errno to ENOMEM, as per the C standard. + errno = ENOMEM; + + // Return nullptr on failure. + return nullptr; + } + }; + } +} // namespace snmalloc + void* operator new(size_t size) { - return snmalloc::libc::malloc(size); + return snmalloc::alloc(size); } void* operator new[](size_t size) { - return snmalloc::libc::malloc(size); + return snmalloc::alloc(size); } -void* operator new(size_t size, std::nothrow_t&) +void* operator new(size_t size, const std::nothrow_t&) noexcept { - return snmalloc::libc::malloc(size); + return snmalloc::alloc(size); } -void* operator new[](size_t size, std::nothrow_t&) +void* operator new[](size_t size, const std::nothrow_t&) noexcept { - return snmalloc::libc::malloc(size); + return snmalloc::alloc(size); } void operator delete(void* p) EXCEPTSPEC @@ -50,7 +126,7 @@ void operator delete(void* p, size_t size) EXCEPTSPEC snmalloc::libc::free_sized(p, size); } -void operator delete(void* p, std::nothrow_t&) +void operator delete(void* p, const std::nothrow_t&) noexcept { snmalloc::libc::free(p); } @@ -65,7 +141,7 @@ void operator delete[](void* p, size_t size) EXCEPTSPEC snmalloc::libc::free_sized(p, size); } -void operator delete[](void* p, std::nothrow_t&) +void operator delete[](void* p, const std::nothrow_t&) noexcept { snmalloc::libc::free(p); } @@ -73,25 +149,27 @@ void operator delete[](void* p, std::nothrow_t&) void* operator new(size_t size, std::align_val_t val) { size = snmalloc::aligned_size(size_t(val), size); - return snmalloc::libc::malloc(size); + return snmalloc::alloc(size); } void* operator new[](size_t size, std::align_val_t val) { size = snmalloc::aligned_size(size_t(val), size); - return snmalloc::libc::malloc(size); + return snmalloc::alloc(size); } -void* operator new(size_t size, std::align_val_t val, std::nothrow_t&) +void* operator new( + size_t size, std::align_val_t val, const std::nothrow_t&) noexcept { size = snmalloc::aligned_size(size_t(val), size); - return snmalloc::libc::malloc(size); + return snmalloc::alloc(size); } -void* operator new[](size_t size, std::align_val_t val, std::nothrow_t&) +void* operator new[]( + size_t size, std::align_val_t val, const std::nothrow_t&) noexcept { size = snmalloc::aligned_size(size_t(val), size); - return snmalloc::libc::malloc(size); + return snmalloc::alloc(size); } void operator delete(void* p, std::align_val_t) EXCEPTSPEC diff --git a/src/test/func/miracle_ptr/miracle_ptr.cc b/src/test/func/miracle_ptr/miracle_ptr.cc index f0504597b..b1f212408 100644 --- a/src/test/func/miracle_ptr/miracle_ptr.cc +++ b/src/test/func/miracle_ptr/miracle_ptr.cc @@ -18,9 +18,27 @@ int main() # include # include # include +# include # include # include +// This code makes the delete overloads build with the correct noexcept spec. +# ifdef _WIN32 +# ifdef __clang__ +# define EXCEPTSPEC noexcept +# else +# define EXCEPTSPEC +# endif +# else +# ifdef _GLIBCXX_USE_NOEXCEPT +# define EXCEPTSPEC _GLIBCXX_USE_NOEXCEPT +# elif defined(_NOEXCEPT) +# define EXCEPTSPEC _NOEXCEPT +# else +# define EXCEPTSPEC +# endif +# endif + namespace snmalloc { // Instantiate the allocator with a client meta data provider that uses an @@ -170,12 +188,12 @@ void* operator new(size_t size) return snmalloc::miracle::malloc(size); } -void operator delete(void* p) +void operator delete(void* p) EXCEPTSPEC { snmalloc::miracle::free(p); } -void operator delete(void* p, size_t) +void operator delete(void* p, size_t) EXCEPTSPEC { snmalloc::miracle::free(p); }