diff --git a/NeoML/Python/src/PyDnnBlob.cpp b/NeoML/Python/src/PyDnnBlob.cpp index 7fc07929b..c6fec80d4 100644 --- a/NeoML/Python/src/PyDnnBlob.cpp +++ b/NeoML/Python/src/PyDnnBlob.cpp @@ -1,4 +1,4 @@ -/* Copyright © 2017-2021 ABBYY Production LLC +/* Copyright © 2017-2024 ABBYY Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -102,7 +102,7 @@ class CPyMemoryHandle : public CMemoryHandle { { } - void* GetPtr() const { return const_cast( CMemoryHandle::object ); } + void* GetPtr() const { return const_cast( CMemoryHandle::Object ); } }; static CBlobDesc createBlobDesc( TBlobType type, std::initializer_list dimensions ) diff --git a/NeoMathEngine/include/NeoMathEngine/MemoryHandle.h b/NeoMathEngine/include/NeoMathEngine/MemoryHandle.h index ebe741e6a..7821ed418 100644 --- a/NeoMathEngine/include/NeoMathEngine/MemoryHandle.h +++ b/NeoMathEngine/include/NeoMathEngine/MemoryHandle.h @@ -1,4 +1,4 @@ -/* Copyright © 2017-2020 ABBYY Production LLC +/* Copyright © 2017-2024 ABBYY Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -27,54 +27,40 @@ class CMemoryHandleInternal; // Wraps the pointer to memory allocated by a math engine class NEOMATHENGINE_API CMemoryHandle { public: - CMemoryHandle() : mathEngine( 0 ), object( 0 ), offset( 0 ) {} - CMemoryHandle( const CMemoryHandle& other ) : mathEngine( other.mathEngine ), object( other.object ), offset( other.offset ) {} - - CMemoryHandle& operator=( const CMemoryHandle& other ) - { - mathEngine = other.mathEngine; - object = other.object; - offset = other.offset; - return *this; - } - + constexpr CMemoryHandle() = default; + // Be copied and moved by default + + bool operator!=( const CMemoryHandle& other ) const { return !operator==( other ); } bool operator==( const CMemoryHandle& other ) const - { - return mathEngine == other.mathEngine && object == other.object && offset == other.offset; - } - - bool operator!=( const CMemoryHandle& other ) const - { - return !operator==( other ); - } + { return MathEngine == other.MathEngine && Object == other.Object && Offset == other.Offset; } - bool IsNull() const - { - return mathEngine == 0 && object == 0 && offset == 0; - } + bool IsNull() const { return *this == CMemoryHandle{}; } - IMathEngine* GetMathEngine() const { return mathEngine; } + IMathEngine* GetMathEngine() const { return MathEngine; } protected: - IMathEngine* mathEngine; // the math engine - const void* object; // the base object - std::ptrdiff_t offset; // the offset in the base object, in bytes + IMathEngine* MathEngine = nullptr; // the math engine owner + const void* Object = nullptr; // the memory allocated base pointer + std::ptrdiff_t Offset = 0; // the offset in the memory allocated volume, in bytes friend class CMemoryHandleInternal; - explicit CMemoryHandle( IMathEngine* _mathEngine, const void* _object, ptrdiff_t _offset ) : mathEngine( _mathEngine ), object( _object ), offset( _offset ) {} + explicit CMemoryHandle( IMathEngine* mathEngine, const void* object, ptrdiff_t offset ) : + MathEngine( mathEngine ), Object( object ), Offset( offset ) {} - CMemoryHandle CopyMemoryHandle( ptrdiff_t shift ) const { return CMemoryHandle( mathEngine, object, offset + shift ); } + CMemoryHandle Copy( ptrdiff_t shift ) const { return CMemoryHandle( MathEngine, Object, Offset + shift ); } }; -//------------------------------------------------------------------------------------------------------------ +//--------------------------------------------------------------------------------------------------------------------- // Wraps the typed pointer to memory allocated by a math engine template class CTypedMemoryHandle : public CMemoryHandle { public: - CTypedMemoryHandle() = default; + constexpr CTypedMemoryHandle() = default; + // Converting ctor explicit CTypedMemoryHandle( const CMemoryHandle& other ) : CMemoryHandle( other ) {} + // Be copied and moved by default void SetValueAt( int index, T value ) const; T GetValueAt( int index ) const; @@ -89,45 +75,45 @@ class CTypedMemoryHandle : public CMemoryHandle { CTypedMemoryHandle& operator+=( ptrdiff_t shift ) { - offset += shift * sizeof( T ); + Offset += shift * sizeof( T ); return *this; } CTypedMemoryHandle& operator-=( ptrdiff_t shift ) { - offset -= shift * sizeof( T ); + Offset -= shift * sizeof( T ); return *this; } CTypedMemoryHandle& operator++() { - offset += sizeof( T ); + Offset += sizeof( T ); return *this; } CTypedMemoryHandle operator++( int ) { CTypedMemoryHandle result( *this ); - offset += sizeof( T ); + Offset += sizeof( T ); return result; } CTypedMemoryHandle& operator--() { - offset -= sizeof( T ); + Offset -= sizeof( T ); return *this; } CTypedMemoryHandle operator--( int ) { CTypedMemoryHandle result( *this ); - offset -= sizeof( T ); + Offset -= sizeof( T ); return result; } CTypedMemoryHandle operator+( ptrdiff_t shift ) const { - return CTypedMemoryHandle( CopyMemoryHandle( shift * sizeof( T ) ) ); + return CTypedMemoryHandle( Copy( shift * sizeof( T ) ) ); } CTypedMemoryHandle operator-( ptrdiff_t shift ) const @@ -137,13 +123,13 @@ class CTypedMemoryHandle : public CMemoryHandle { int operator-( const CTypedMemoryHandle& handle ) const { - return ( int ) ( offset - handle.offset ); + return ( int ) ( Offset - handle.Offset ); } }; -//------------------------------------------------------------------------------------------------------------ +//--------------------------------------------------------------------------------------------------------------------- -// CMemoryHandleVar is a variable or fixed-size array for a math engine +// CMemoryHandleVar is a variable or a fixed-size array for a math engine template class CMemoryHandleVarBase { public: @@ -152,7 +138,7 @@ class CMemoryHandleVarBase { void SetValue( T value ); T GetValue() const; - const CTypedMemoryHandle& GetHandle() const { return data; } + const CTypedMemoryHandle& GetHandle() const { return Data; } // Operators for easier use operator const CTypedMemoryHandle&( ) const { return GetHandle(); } @@ -163,44 +149,40 @@ class CMemoryHandleVarBase { bool operator!=( const CTypedMemoryHandle& other ) const { return GetHandle() != other; } CTypedMemoryHandle operator+( ptrdiff_t shift ) const { return GetHandle() + shift; } - CTypedMemoryHandle operator-( ptrdiff_t shift ) const { return GetHandle() - shift; } - int operator-( const CTypedMemoryHandle& handle ) const { return GetHandle() - handle; } - int Size() const { return static_cast( size ); } - - IMathEngine* GetMathEngine() const { return mathEngine; } + int Size() const { return static_cast( DataSize ); } + IMathEngine* GetMathEngine() const { return &MathEngine; } protected: - CMemoryHandleVarBase( IMathEngine& _mathEngine, size_t _size ) : mathEngine( &_mathEngine ), size( _size ) {} + IMathEngine& MathEngine; // the math engine owner + mutable CTypedMemoryHandle Data; // the typed memory handler + const size_t DataSize; // the typed memory size - mutable IMathEngine* mathEngine; - mutable CTypedMemoryHandle data; - const size_t size; + CMemoryHandleVarBase( IMathEngine& mathEngine, size_t size ) : MathEngine( mathEngine ), DataSize( size ) {} + ~CMemoryHandleVarBase() = default; private: // may not be copied, only passed by reference - CMemoryHandleVarBase( const CMemoryHandleVarBase& ); - CMemoryHandleVarBase& operator=( const CMemoryHandleVarBase& ); + CMemoryHandleVarBase( const CMemoryHandleVarBase& ) = delete; + CMemoryHandleVarBase& operator=( const CMemoryHandleVarBase& ) = delete; }; -//------------------------------------------------------------------------------------------------------------ +//--------------------------------------------------------------------------------------------------------------------- -// A variable or array +// A variable or an array template class CMemoryHandleVar : public CMemoryHandleVarBase { public: explicit CMemoryHandleVar( IMathEngine& mathEngine, size_t size = 1 ); ~CMemoryHandleVar(); - - const CTypedMemoryHandle& GetHandle() const { return CMemoryHandleVarBase::GetHandle(); } }; -//------------------------------------------------------------------------------------------------------------ +//--------------------------------------------------------------------------------------------------------------------- -// A variable or array on stack +// A variable or an array on the stack template class CMemoryHandleStackVar : public CMemoryHandleVarBase { public: @@ -209,8 +191,10 @@ class CMemoryHandleStackVar : public CMemoryHandleVarBase { ~CMemoryHandleStackVar(); }; -//------------------------------------------------------------------------------------------------------------ +//--------------------------------------------------------------------------------------------------------------------- + // typedefs + typedef CTypedMemoryHandle CFloatHandle; typedef CTypedMemoryHandle CConstFloatHandle; diff --git a/NeoMathEngine/include/NeoMathEngine/MemoryHandle.inl b/NeoMathEngine/include/NeoMathEngine/MemoryHandle.inl index b31518517..12a2b597c 100644 --- a/NeoMathEngine/include/NeoMathEngine/MemoryHandle.inl +++ b/NeoMathEngine/include/NeoMathEngine/MemoryHandle.inl @@ -1,4 +1,4 @@ -/* Copyright © 2017-2020 ABBYY Production LLC +/* Copyright © 2017-2024 ABBYY Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -23,7 +23,7 @@ template inline void CTypedMemoryHandle::SetValueAt( int index, T value ) const { CTypedMemoryHandle result = *this + index; - mathEngine->DataExchangeRaw( result, &value, sizeof( T ) ); + MathEngine->DataExchangeRaw( result, &value, sizeof( T ) ); } template @@ -31,7 +31,7 @@ inline T CTypedMemoryHandle::GetValueAt( int index ) const { char result[sizeof(T)]; CTypedMemoryHandle source = *this + index; - mathEngine->DataExchangeRaw( result, source, sizeof( T ) ); + MathEngine->DataExchangeRaw( result, source, sizeof( T ) ); T* value = reinterpret_cast( &result ); return *value; } @@ -39,14 +39,14 @@ inline T CTypedMemoryHandle::GetValueAt( int index ) const template inline void CTypedMemoryHandle::SetValue( T value ) const { - mathEngine->DataExchangeRaw( *this, &value, sizeof( T ) ); + MathEngine->DataExchangeRaw( *this, &value, sizeof( T ) ); } template inline T CTypedMemoryHandle::GetValue() const { char result[sizeof(T)]; - mathEngine->DataExchangeRaw( result, *this, sizeof( T ) ); + MathEngine->DataExchangeRaw( result, *this, sizeof( T ) ); T* value = reinterpret_cast( &result ); return *value; } @@ -57,25 +57,25 @@ inline T CTypedMemoryHandle::GetValue() const template inline void CMemoryHandleVarBase::SetValueAt( int index, T value ) { - data.SetValueAt( index, value ); + Data.SetValueAt( index, value ); } template inline T CMemoryHandleVarBase::GetValueAt( int index ) const { - return data.GetValueAt( index ); + return Data.GetValueAt( index ); } template inline void CMemoryHandleVarBase::SetValue( T value ) { - data.SetValue( value ); + Data.SetValue( value ); } template inline T CMemoryHandleVarBase::GetValue() const { - return data.GetValue(); + return Data.GetValue(); } //------------------------------------------------------------------------------------------------------------ @@ -86,15 +86,15 @@ inline CMemoryHandleVar::CMemoryHandleVar( IMathEngine& mathEngine, size_t si CMemoryHandleVarBase( mathEngine, size ) { if( size != 0 ) { - CMemoryHandleVarBase::data = CMemoryHandleVarBase::mathEngine->template HeapAllocTyped( size ); + CMemoryHandleVarBase::Data = CMemoryHandleVarBase::MathEngine.template HeapAllocTyped( size ); } } template inline CMemoryHandleVar::~CMemoryHandleVar() { - if( !CMemoryHandleVarBase::data.IsNull() ) { - CMemoryHandleVarBase::mathEngine->HeapFree( CMemoryHandleVarBase::data ); + if( !CMemoryHandleVarBase::Data.IsNull() ) { + CMemoryHandleVarBase::MathEngine.HeapFree( CMemoryHandleVarBase::Data ); } } @@ -105,16 +105,16 @@ inline CMemoryHandleStackVar::CMemoryHandleStackVar( IMathEngine& mathEngine, CMemoryHandleVarBase( mathEngine, size ) { if( size != 0 ) { - CMemoryHandleVarBase::data = - CTypedMemoryHandle( CMemoryHandleVarBase::mathEngine->StackAlloc( size * sizeof(T) ) ); + CMemoryHandleVarBase::Data = + CTypedMemoryHandle( CMemoryHandleVarBase::MathEngine.StackAlloc( size * sizeof( T ) ) ); } } template inline CMemoryHandleStackVar::~CMemoryHandleStackVar() { - if( !CMemoryHandleVarBase::data.IsNull() ) { - CMemoryHandleVarBase::mathEngine->StackFree( CMemoryHandleVarBase::data ); + if( !CMemoryHandleVarBase::Data.IsNull() ) { + CMemoryHandleVarBase::MathEngine.StackFree( CMemoryHandleVarBase::Data ); } } diff --git a/NeoMathEngine/src/CMakeLists.txt b/NeoMathEngine/src/CMakeLists.txt index 90a7d71a9..2902c91b6 100644 --- a/NeoMathEngine/src/CMakeLists.txt +++ b/NeoMathEngine/src/CMakeLists.txt @@ -27,10 +27,9 @@ set(CPU_COMMON_SOURCES CPU/CpuMathEngineVectorMath.cpp CrtAllocatedObject.cpp DllLoader.cpp - MathEngineDeviceStackAllocator.cpp MathEngineDnnDropout.cpp MathEngine.cpp - MathEngineHostStackAllocator.cpp + MathEngineStackAllocator.cpp MemoryEngineMixin.cpp MemoryPool.cpp ThreadPool.cpp @@ -44,13 +43,12 @@ target_sources(${PROJECT_NAME} common.h MathEngineAllocator.h MathEngineCommon.h - MathEngineDeviceStackAllocator.h MathEngineDll.h MathEngineDnnConv.h MathEngineDnnDropout.h MathEngineDnnLrn.h MathEngineDnnPoolings.h - MathEngineHostStackAllocator.h + MathEngineStackAllocator.h MemoryEngineMixin.h MemoryHandleInternal.h MemoryPool.h @@ -104,7 +102,7 @@ set_target_properties( ${PROJECT_NAME} PROPERTIES UNITY_BUILD_MODE GROUP ) set_property(SOURCE ${CPU_COMMON_SOURCES} PROPERTY UNITY_GROUP 1) -set_property(SOURCE MathEngineDeviceStackAllocator.cpp PROPERTY SKIP_UNITY_BUILD_INCLUSION ON) +set_property(SOURCE MathEngineStackAllocator.cpp PROPERTY SKIP_UNITY_BUILD_INCLUSION ON) target_compile_features(${PROJECT_NAME} PUBLIC cxx_std_14) diff --git a/NeoMathEngine/src/CPU/CpuMathEngine.cpp b/NeoMathEngine/src/CPU/CpuMathEngine.cpp index 8e82a4132..1ffcce2de 100644 --- a/NeoMathEngine/src/CPU/CpuMathEngine.cpp +++ b/NeoMathEngine/src/CPU/CpuMathEngine.cpp @@ -17,8 +17,6 @@ limitations under the License. #pragma hdrstop #include -#include -#include #include #include #include diff --git a/NeoMathEngine/src/GPU/CUDA/CudaMathEngine.cpp b/NeoMathEngine/src/GPU/CUDA/CudaMathEngine.cpp index 6c173a307..fb6ff0dca 100644 --- a/NeoMathEngine/src/GPU/CUDA/CudaMathEngine.cpp +++ b/NeoMathEngine/src/GPU/CUDA/CudaMathEngine.cpp @@ -26,8 +26,6 @@ limitations under the License. #include #include #include -#include -#include #include #include #include diff --git a/NeoMathEngine/src/GPU/CUDA/CudaMathEngine.cu b/NeoMathEngine/src/GPU/CUDA/CudaMathEngine.cu index 3beb99b6a..910788a56 100644 --- a/NeoMathEngine/src/GPU/CUDA/CudaMathEngine.cu +++ b/NeoMathEngine/src/GPU/CUDA/CudaMathEngine.cu @@ -25,8 +25,6 @@ limitations under the License. #include #include #include -#include -#include #include #include #include diff --git a/NeoMathEngine/src/GPU/Metal/MetalMathEngine.mm b/NeoMathEngine/src/GPU/Metal/MetalMathEngine.mm index 6b70be7fc..4d9fc9d0c 100644 --- a/NeoMathEngine/src/GPU/Metal/MetalMathEngine.mm +++ b/NeoMathEngine/src/GPU/Metal/MetalMathEngine.mm @@ -26,7 +26,6 @@ @import std.vector; #include -#include @import Foundation; @import MetalKit; diff --git a/NeoMathEngine/src/GPU/Vulkan/VulkanMathEngine.cpp b/NeoMathEngine/src/GPU/Vulkan/VulkanMathEngine.cpp index 227e422fa..5f6eea5ce 100644 --- a/NeoMathEngine/src/GPU/Vulkan/VulkanMathEngine.cpp +++ b/NeoMathEngine/src/GPU/Vulkan/VulkanMathEngine.cpp @@ -24,8 +24,6 @@ limitations under the License. #include #include #include -#include -#include #include #include #include diff --git a/NeoMathEngine/src/MathEngineDeviceStackAllocator.cpp b/NeoMathEngine/src/MathEngineDeviceStackAllocator.cpp deleted file mode 100644 index 1df8b44c1..000000000 --- a/NeoMathEngine/src/MathEngineDeviceStackAllocator.cpp +++ /dev/null @@ -1,233 +0,0 @@ -/* Copyright © 2017-2020 ABBYY Production LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. ---------------------------------------------------------------------------------------------------------------*/ - -#include -#pragma hdrstop - -#include -#include -#include - -namespace NeoML { - -// The memory manager for stack allocation. Tries to keep all allocated memory in one block -static const int StackBlockQuantum = 64 * 1024; // 64 K - -// Device memory block used for the stack -class CDeviceStackBlock : public CCrtAllocatedObject { -public: - CDeviceStackBlock( CMemoryPool& _memoryManager, size_t size, CDeviceStackBlock* prev ) : - Prev(prev), - memoryManager( _memoryManager ), - blockSize( ( size + StackBlockQuantum - 1 ) / StackBlockQuantum * StackBlockQuantum ), - allocSize(0) - { - buffer = CTypedMemoryHandle( memoryManager.Alloc( blockSize ) ); - } - - ~CDeviceStackBlock() { memoryManager.Free( buffer ); } - - size_t GetBlockSize() const { return blockSize; } - - size_t GetAllocSize() const { return allocSize; } - - CMemoryHandle TryAlloc(size_t size) - { - size_t newAllocSize = allocSize + size; - if( newAllocSize > blockSize ) { - return CMemoryHandle(); - } - - CMemoryHandle result = buffer + allocSize; - allocSize = newAllocSize; - - return result; - } - - CMemoryHandle Alloc( size_t size ) - { - CMemoryHandle res = TryAlloc(size); - PRESUME_EXPR( !res.IsNull() ); - return res; - } - - // Returns the size of released block - size_t Free( const CMemoryHandle& ptr ) - { - ptrdiff_t diff = CTypedMemoryHandle( ptr ) - buffer; - PRESUME_EXPR(0 <= diff && diff < (ptrdiff_t)blockSize); // the pointer belongs to this block - - size_t ret = allocSize - diff; - allocSize = diff; - - return ret; - } - - CDeviceStackBlock* const Prev; - -private: - CMemoryPool& memoryManager; - const size_t blockSize; - size_t allocSize; - CTypedMemoryHandle buffer; -}; - -//------------------------------------------------------------------------------------------------------------ - -// The manager class -class CDeviceStackMemoryManager : public CCrtAllocatedObject { -public: - explicit CDeviceStackMemoryManager( CMemoryPool& _memoryManager ) : - memoryManager( _memoryManager ), - head(0), - maxAllocSize(0), - curAllocSize(0) - { - } - - ~CDeviceStackMemoryManager() - { - cleanUpWorker(); - } - - void CleanUp() - { - PRESUME_EXPR( head == 0 || ( head->Prev == 0 && head->GetAllocSize() == 0 ) ); - cleanUpWorker(); - } - - CMemoryHandle Alloc(size_t size) - { - curAllocSize += size; - if(maxAllocSize < curAllocSize) { - maxAllocSize = curAllocSize; - } - - if( head == 0 || ( head->Prev == 0 && head->GetBlockSize() < maxAllocSize && head->GetAllocSize() == 0 ) ) { - // Allocate a new block for all required memory - if(head != 0) { - delete head; - } - head = new CDeviceStackBlock( memoryManager, maxAllocSize, 0 ); - return head->Alloc(size); - } - - // Try to allocate space in the current block - CMemoryHandle res = head->TryAlloc(size); - if( !res.IsNull() ) { - return res; - } - - // Create a new block - head = new CDeviceStackBlock( memoryManager, size, head ); - return head->Alloc(size); - } - - void Free( const CMemoryHandle& ptr ) - { - PRESUME_EXPR(head != 0); - - size_t size = head->Free(ptr); - PRESUME_EXPR(size <= curAllocSize); - - curAllocSize -= size; - - if(head->GetAllocSize() == 0 && head->Prev != 0) { - CDeviceStackBlock* blockToDelete = head; - head = head->Prev; - delete blockToDelete; - } - } - -private: - CMemoryPool& memoryManager; - CDeviceStackBlock *head; - size_t maxAllocSize; - size_t curAllocSize; - - void cleanUpWorker() - { - while(head != 0) { - CDeviceStackBlock* blockToDelete = head; - head = head->Prev; - delete blockToDelete; - } - maxAllocSize = 0; - curAllocSize = 0; - } - -}; - -//------------------------------------------------------------------------------------------------------------ - -CDeviceStackAllocator::CDeviceStackAllocator( CMemoryPool& _memoryPool, int _memoryAlignment ) : - memoryPool( _memoryPool ), - memoryAlignment( _memoryAlignment ) -{ -} - -CDeviceStackAllocator::~CDeviceStackAllocator() -{ - for( auto cur : stackManagers ) { - delete cur.second; - } -} - -void CDeviceStackAllocator::CleanUp() -{ - std::thread::id id = std::this_thread::get_id(); - - auto iterator = stackManagers.find( id ); - if( iterator != stackManagers.end() ) { - iterator->second->CleanUp(); - } -} - -CMemoryHandle CDeviceStackAllocator::Alloc( size_t size ) -{ - // Align size to keep correct data alignment - size = ( ( size + memoryAlignment - 1 ) / memoryAlignment ) * memoryAlignment; - CDeviceStackMemoryManager* deviceManager = 0; - std::thread::id id = std::this_thread::get_id(); - - { - auto result = stackManagers.find( id ); - if( result == stackManagers.end() ) { - result = stackManagers.insert( make_pair( id, new CDeviceStackMemoryManager( memoryPool ) ) ).first; - } - deviceManager = result->second; - } - - return deviceManager->Alloc(size); -} - -void CDeviceStackAllocator::Free( const CMemoryHandle& ptr ) -{ - if( ptr.IsNull() ) { - return; - } - - CDeviceStackMemoryManager* deviceManager = 0; - std::thread::id id = std::this_thread::get_id(); - - { - deviceManager = stackManagers.find( id )->second; - } - - deviceManager->Free(ptr); -} - -} // namespace NeoML - diff --git a/NeoMathEngine/src/MathEngineDeviceStackAllocator.h b/NeoMathEngine/src/MathEngineDeviceStackAllocator.h deleted file mode 100644 index a9304349d..000000000 --- a/NeoMathEngine/src/MathEngineDeviceStackAllocator.h +++ /dev/null @@ -1,50 +0,0 @@ -/* Copyright © 2017-2020 ABBYY Production LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. ---------------------------------------------------------------------------------------------------------------*/ - -#pragma once - -#include -#include -#include -#include -#include -#include -#include - - -namespace NeoML { - -class CDeviceStackMemoryManager; - -// Device memory stack implementation for MathEngine -class CDeviceStackAllocator : public CCrtAllocatedObject { -public: - CDeviceStackAllocator( CMemoryPool& memoryPool, int memoryAlignment ); - ~CDeviceStackAllocator(); - - void CleanUp(); - - CMemoryHandle Alloc( size_t size ); - void Free( const CMemoryHandle& ptr ); - -private: - CMemoryPool& memoryPool; - const int memoryAlignment; - std::unordered_map< std::thread::id, CDeviceStackMemoryManager*, - std::hash, std::equal_to, - CrtAllocator< std::pair > > stackManagers; -}; - -} // namespace NeoML diff --git a/NeoMathEngine/src/MathEngineHostStackAllocator.cpp b/NeoMathEngine/src/MathEngineHostStackAllocator.cpp deleted file mode 100644 index 1a76b78ce..000000000 --- a/NeoMathEngine/src/MathEngineHostStackAllocator.cpp +++ /dev/null @@ -1,226 +0,0 @@ -/* Copyright © 2017-2020 ABBYY Production LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. ---------------------------------------------------------------------------------------------------------------*/ - -#include -#pragma hdrstop - -#include -#include - -namespace NeoML { - -// The memory manager for stack allocation. Tries to keep all allocated memory in one block -static const int StackBlockQuantum = 64 * 1024; // 64 K - -// Host memory block used for the stack -class CHostStackBlock : public CCrtAllocatedObject { -public: - CHostStackBlock( size_t size, CHostStackBlock* prev ) : - Prev(prev), - blockSize( ( size + StackBlockQuantum - 1 ) / StackBlockQuantum * StackBlockQuantum ), - allocSize(0) - { - buffer = reinterpret_cast( malloc( blockSize ) ); - } - - ~CHostStackBlock() { free( buffer ); } - - size_t GetBlockSize() const { return blockSize; } - - size_t GetAllocSize() const { return allocSize; } - - void* TryAlloc( size_t size ) - { - size_t newAllocSize = allocSize + size; - if( newAllocSize > blockSize ) { - return 0; - } - - char* result = buffer + allocSize; - allocSize = newAllocSize; - - return result; - } - - void* Alloc( size_t size ) - { - void* res = TryAlloc(size); - PRESUME_EXPR( res != 0 ); - return res; - } - - // Returns the size of released block - size_t Free( void* ptr ) - { - ptrdiff_t diff = reinterpret_cast( ptr ) - buffer; - PRESUME_EXPR(0 <= diff && diff < (ptrdiff_t)blockSize); // the pointer belongs to this block - - size_t ret = allocSize - diff; - allocSize = diff; - - return ret; - } - - CHostStackBlock* const Prev; - -private: - const size_t blockSize; - size_t allocSize; - char* buffer; -}; - -//------------------------------------------------------------------------------------------------------------ - -// The manager class -class CHostStackMemoryManager : public CCrtAllocatedObject { -public: - CHostStackMemoryManager() = default; - - ~CHostStackMemoryManager() - { - cleanUpWorker(); - } - - void CleanUp() - { - PRESUME_EXPR( head == 0 || ( head->Prev == 0 && head->GetAllocSize() == 0 ) ); - cleanUpWorker(); - } - - void* Alloc(size_t size) - { - curAllocSize += size; - if(maxAllocSize < curAllocSize) { - maxAllocSize = curAllocSize; - } - - if( head == 0 || ( head->Prev == 0 && head->GetBlockSize() < maxAllocSize && head->GetAllocSize() == 0 ) ) { - // Allocate a new block for all required memory - if(head != 0) { - delete head; - } - head = new CHostStackBlock( maxAllocSize, 0 ); - return head->Alloc(size); - } - - // Try to allocate space in the current block - void* res = head->TryAlloc(size); - if( res != 0 ) { - return res; - } - - // Create a new block - head = new CHostStackBlock( size, head ); - return head->Alloc(size); - } - - void Free( void* ptr ) - { - PRESUME_EXPR(head != 0); - - size_t size = head->Free(ptr); - PRESUME_EXPR(size <= curAllocSize); - - curAllocSize -= size; - - if( head->GetAllocSize() == 0 && head->Prev != 0 ) { - CHostStackBlock* blockToDelete = head; - head = head->Prev; - delete blockToDelete; - } - } - -private: - CHostStackBlock *head{}; - size_t maxAllocSize{}; - size_t curAllocSize{}; - - void cleanUpWorker() - { - while( head != 0 ) { - CHostStackBlock* blockToDelete = head; - head = head->Prev; - delete blockToDelete; - } - maxAllocSize = 0; - curAllocSize = 0; - } -}; - -//------------------------------------------------------------------------------------------------------------ - -CHostStackAllocator::CHostStackAllocator( int _memoryAlignment ) : - memoryAlignment( _memoryAlignment ) -{ -} - -CHostStackAllocator::~CHostStackAllocator() -{ - for( auto cur : stackManagers ) { - delete cur.second; - } -} - -void CHostStackAllocator::CleanUp() -{ - std::thread::id id = std::this_thread::get_id(); - - std::lock_guard lock( mutex ); - auto iterator = stackManagers.find( id ); - if( iterator != stackManagers.end() ) { - iterator->second->CleanUp(); - } -} - -void* CHostStackAllocator::Alloc( size_t size ) -{ - // Align size to keep correct data alignment - size = ( ( size + memoryAlignment - 1 ) / memoryAlignment ) * memoryAlignment; - CHostStackMemoryManager* hostManager = 0; - std::thread::id id = std::this_thread::get_id(); - - { - std::lock_guard lock( mutex ); - auto result = stackManagers.find( id ); - if( result == stackManagers.end() ) { - result = stackManagers.insert( make_pair( id, new CHostStackMemoryManager() ) ).first; - } - hostManager = result->second; - } - - return hostManager->Alloc(size); -} - -void CHostStackAllocator::Free( void* ptr ) -{ - if( ptr == 0 ) { - return; - } - - CHostStackMemoryManager* hostManager = 0; - std::thread::id id = std::this_thread::get_id(); - - { - std::lock_guard lock( mutex ); - - auto pair = stackManagers.find( id ); - hostManager = pair->second; - } - - hostManager->Free(ptr); -} - -} // namespace NeoML - diff --git a/NeoMathEngine/src/MathEngineHostStackAllocator.h b/NeoMathEngine/src/MathEngineHostStackAllocator.h deleted file mode 100644 index bd74d1968..000000000 --- a/NeoMathEngine/src/MathEngineHostStackAllocator.h +++ /dev/null @@ -1,49 +0,0 @@ -/* Copyright © 2017-2020 ABBYY Production LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. ---------------------------------------------------------------------------------------------------------------*/ - -#pragma once - -#include -#include -#include -#include -#include -#include - - -namespace NeoML { - -class CHostStackMemoryManager; - -// Host memory stack implementation for MathEngine -class CHostStackAllocator : public CCrtAllocatedObject { -public: - explicit CHostStackAllocator( int memoryAlignment ); - ~CHostStackAllocator(); - - void CleanUp(); - - void* Alloc( size_t size ); - void Free( void* ptr ); - -private: - const int memoryAlignment; - std::mutex mutex; - std::unordered_map< std::thread::id, CHostStackMemoryManager*, - std::hash, std::equal_to, - CrtAllocator< std::pair > > stackManagers; -}; - -} // namespace NeoML diff --git a/NeoMathEngine/src/MathEngineStackAllocator.cpp b/NeoMathEngine/src/MathEngineStackAllocator.cpp new file mode 100644 index 000000000..4f323dc22 --- /dev/null +++ b/NeoMathEngine/src/MathEngineStackAllocator.cpp @@ -0,0 +1,336 @@ +/* Copyright © 2024 ABBYY + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--------------------------------------------------------------------------------------------------------------*/ + +#include +#pragma hdrstop + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace NeoML { + +static_assert( int( TStackAlloc::Count_ ) == 2, "TStackAlloc: Host and Device are allowed only" ); + +// Host +CStackMemoryHandle::CStackMemoryHandle( void* host ) : + handle( CMemoryHandleInternal::CreateMemoryHandle( /*mathEngine*/nullptr, host ) ) +{ +} + +// Device +CStackMemoryHandle::CStackMemoryHandle( CMemoryHandle device ) : + handle( device ) +{ + PRESUME_EXPR( handle.GetMathEngine() != nullptr ); +} + +// Host +CStackMemoryHandle::operator void*() const +{ + ASSERT_EXPR( handle.GetMathEngine() == nullptr ); + return GetRaw( handle ); +} + +// Device +CStackMemoryHandle::operator CMemoryHandle() const +{ + ASSERT_EXPR( handle.GetMathEngine() != nullptr ); + return handle; +} + +CStackMemoryHandle CStackMemoryHandle::operator+( size_t size ) const +{ + PRESUME_EXPR( size <= size_t( PTRDIFF_MAX ) ); + return CStackMemoryHandle( CTypedMemoryHandle( handle ) + size ); +} + +int CStackMemoryHandle::operator-( const CStackMemoryHandle& ptr ) const +{ + return CTypedMemoryHandle( handle ) - CTypedMemoryHandle( ptr.handle ); +} + +//--------------------------------------------------------------------------------------------------------------------- + +class CStackAllocManager final : public CCrtAllocatedObject { +public: + explicit CStackAllocManager( CMemoryPool* memoryPool ) : memoryPool( memoryPool ) {} + + TStackAlloc Type() const { return ( memoryPool == nullptr ) ? TStackAlloc::Host : TStackAlloc::Device; } + + CStackMemoryHandle Alloc( size_t size ); + void Free( const CStackMemoryHandle& handle ); + +private: + CMemoryPool* const memoryPool; +}; + +CStackMemoryHandle CStackAllocManager::Alloc( size_t size ) +{ + return ( memoryPool != nullptr ) + ? CStackMemoryHandle( CMemoryHandle( memoryPool->Alloc( size ) ) ) + : CStackMemoryHandle( malloc( size ) ); +} + +void CStackAllocManager::Free( const CStackMemoryHandle& ptr ) +{ + PRESUME_EXPR( !ptr.IsNull() ); + ( memoryPool != nullptr ) + ? memoryPool->Free( ptr ) + : free( static_cast( ptr ) ); +} + +//--------------------------------------------------------------------------------------------------------------------- + +// Device memory block used for the stack +class CStackBlock final : public CCrtAllocatedObject { +public: + CStackBlock( CStackAllocManager& allocManager, size_t size, CStackBlock* prev ); + ~CStackBlock() { manager.Free( buffer ); } + + size_t GetBlockSize() const { return blockSize; } + size_t GetAllocSize() const { return allocSize; } + // One-dimensional list + CStackBlock* Previous() const { return previous; } + + CStackMemoryHandle TryAlloc( size_t size ); + CStackMemoryHandle Alloc( size_t size ); + // Returns the size of released block + size_t Free( const CStackMemoryHandle& ptr ); + +private: + // The memory manager for stack allocation. Tries to keep all allocated memory in one block + static constexpr int quantum = 64 * 1024; // 64 KB + + CStackAllocManager& manager; + const size_t blockSize; + size_t allocSize; + const CStackMemoryHandle buffer; + CStackBlock* const previous; // One-dimensional list +}; + +CStackBlock::CStackBlock( CStackAllocManager& allocManager, size_t size, CStackBlock* prev ) : + manager( allocManager ), + blockSize( ( size + quantum - 1 ) / quantum * quantum ), + allocSize( 0 ), + buffer( manager.Alloc( blockSize ) ), + previous( prev ) +{} + +CStackMemoryHandle CStackBlock::TryAlloc( size_t size ) +{ + const size_t newAllocSize = allocSize + size; + if( newAllocSize > blockSize ) { + return CStackMemoryHandle{}; + } + + const CStackMemoryHandle result = buffer + allocSize; + allocSize = newAllocSize; + return result; +} + +CStackMemoryHandle CStackBlock::Alloc( size_t size ) +{ + const CStackMemoryHandle result = TryAlloc( size ); + PRESUME_EXPR( !result.IsNull() ); + return result; +} + +size_t CStackBlock::Free( const CStackMemoryHandle& ptr ) +{ + const int diff = ptr - buffer; + PRESUME_EXPR( 0 <= diff && diff < static_cast( blockSize ) ); // the pointer belongs to this block + + const size_t result = allocSize - diff; + allocSize = diff; + return result; +} + +//--------------------------------------------------------------------------------------------------------------------- + +// The manager class +class CStackBlockManager final : public CCrtAllocatedObject { +public: + explicit CStackBlockManager( CStackAllocManager& allocManager ) : manager( allocManager ) {} + ~CStackBlockManager() { cleanUpWorker(); } + + void CleanUp(); + + CStackMemoryHandle Alloc( size_t size ); + void Free( const CStackMemoryHandle& ptr ); + +private: + CStackAllocManager& manager; + CStackBlock* head = nullptr; + size_t maxAllocSize = 0; + size_t curAllocSize = 0; + + void cleanUpWorker(); +}; + +void CStackBlockManager::CleanUp() +{ + PRESUME_EXPR( head == nullptr || ( head->Previous() == nullptr && head->GetAllocSize() == 0 ) ); + cleanUpWorker(); +} + +CStackMemoryHandle CStackBlockManager::Alloc( size_t size ) +{ + curAllocSize += size; + if( maxAllocSize < curAllocSize ) { + maxAllocSize = curAllocSize; + } + + if( head == nullptr + || ( head->Previous() == nullptr && head->GetBlockSize() < maxAllocSize && head->GetAllocSize() == 0 ) ) + { + // Allocate a new block for all required memory + if( head != nullptr ) { + head->~CStackBlock(); + ::new( head ) CStackBlock( manager, maxAllocSize, nullptr ); + } else { + head = new CStackBlock( manager, maxAllocSize, nullptr ); + } + return head->Alloc( size ); + } + + // Try to allocate space in the current block + CStackMemoryHandle result = head->TryAlloc( size ); + if( !result.IsNull() ) { + return result; + } + + // Create a new block + head = new CStackBlock( manager, size, head ); + return head->Alloc( size ); +} + +void CStackBlockManager::Free( const CStackMemoryHandle& ptr ) +{ + PRESUME_EXPR( head != nullptr ); + + const size_t size = head->Free( ptr ); + PRESUME_EXPR( size <= curAllocSize ); + + curAllocSize -= size; + + // Delete all free blocks except last free block, to reuse it at the next allocation + if( head->GetAllocSize() == 0 && head->Previous() != nullptr ) { + CStackBlock* blockToDelete = head; + head = head->Previous(); + delete blockToDelete; + } +} + +void CStackBlockManager::cleanUpWorker() +{ + while( head != nullptr ) { + CStackBlock* blockToDelete = head; + head = head->Previous(); + delete blockToDelete; + } + maxAllocSize = 0; + curAllocSize = 0; +} + +//--------------------------------------------------------------------------------------------------------------------- + +// Device or Host memory stack implementation for MathEngine +class CStackAllocator : public IStackAllocator { +public: + CStackAllocator( CMemoryPool* memoryPool, int memoryAlignment ); + + TStackAlloc Type() const override { return manager.Type(); } + void CleanUp() override; + + CStackMemoryHandle Alloc( size_t size ) override; + void Free( const CStackMemoryHandle& ptr ) override; + +private: + using TStackBlockManagers = std::unordered_map< + std::thread::id, CStackBlockManager, // (key, value) + std::hash, + std::equal_to, + CrtAllocator> + >; + + CStackAllocManager manager; + const int memoryAlignment; + TStackBlockManagers stackManagers; +}; + +CStackAllocator::CStackAllocator( CMemoryPool* _memoryPool, int _memoryAlignment ) : + manager( _memoryPool ), + memoryAlignment( _memoryAlignment ) +{ +} + +void CStackAllocator::CleanUp() +{ + auto it = stackManagers.find( std::this_thread::get_id() ); + if( it != stackManagers.end() ) { + it->second.CleanUp(); + } +} + +CStackMemoryHandle CStackAllocator::Alloc( size_t size ) +{ + // Align size to keep correct data alignment + size = ( ( size + memoryAlignment - 1 ) / memoryAlignment ) * memoryAlignment; + + std::thread::id id = std::this_thread::get_id(); + auto it = stackManagers.find( id ); + if( it == stackManagers.end() ) { + it = stackManagers.emplace( id, CStackBlockManager( manager ) ).first; + } + CStackBlockManager& manager = it->second; + return manager.Alloc(size); +} + +void CStackAllocator::Free( const CStackMemoryHandle& ptr ) +{ + if( ptr.IsNull() ) { + return; + } + + std::thread::id id = std::this_thread::get_id(); + CStackBlockManager& manager = stackManagers.find( id )->second; + manager.Free(ptr); +} + +//--------------------------------------------------------------------------------------------------------------------- + +IStackAllocator* CreateStackAllocator( TStackAlloc type, CMemoryPool* memoryPool, int memoryAlignment ) +{ + ASSERT_EXPR( memoryAlignment > 0 ); + ASSERT_EXPR( ( type == TStackAlloc::Host && memoryPool == nullptr ) + || ( type == TStackAlloc::Device && memoryPool != nullptr ) ); + return new CStackAllocator( memoryPool, memoryAlignment ); +} + +void CStackAllocatorDeleter::operator()( IStackAllocator* allocator ) const +{ + delete allocator; +} + +} // namespace NeoML + diff --git a/NeoMathEngine/src/MathEngineStackAllocator.h b/NeoMathEngine/src/MathEngineStackAllocator.h new file mode 100644 index 000000000..b5cc2cef9 --- /dev/null +++ b/NeoMathEngine/src/MathEngineStackAllocator.h @@ -0,0 +1,74 @@ +/* Copyright © 2024 ABBYY + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--------------------------------------------------------------------------------------------------------------*/ + +#pragma once + +#include +#include + +namespace NeoML { + +class CMemoryPool; + +// Device or Host memory stack allocator type +enum class TStackAlloc { Host, Device, Count_ }; + +// Stack pointer wrapper for Device or Host memory block +class CStackMemoryHandle final { +public: + // Convert ctors + CStackMemoryHandle( void* host = nullptr ); // default ctor + CStackMemoryHandle( CMemoryHandle device ); + // Be copied and moved by default + + bool IsNull() const { return handle.IsNull(); } + // Types casts + operator void*() const; // host + operator CMemoryHandle() const; // device + + CStackMemoryHandle operator+( size_t size ) const; + int operator-( const CStackMemoryHandle& ptr ) const; + +private: + const CMemoryHandle handle; + + CStackMemoryHandle( CTypedMemoryHandle&& ptr ) : handle( ptr ) {} +}; + +//------------------------------------------------------------------------------------------------------------ + +// Device or Host memory stack allocator implementation for MathEngine +class IStackAllocator : public CCrtAllocatedObject { +public: + virtual ~IStackAllocator() = default; + + virtual TStackAlloc Type() const = 0; + virtual void CleanUp() = 0; + + virtual CStackMemoryHandle Alloc( size_t size ) = 0; + virtual void Free( const CStackMemoryHandle& ptr ) = 0; +}; + +//------------------------------------------------------------------------------------------------------------ + +// Either a memoryPool is equal 0, the stack allocator for the Host memory is crated, or for the Device memory +IStackAllocator* CreateStackAllocator( TStackAlloc type, CMemoryPool* memoryPool, int memoryAlignment ); + +// Deleter for smart pointers +struct CStackAllocatorDeleter final : public CCrtAllocatedObject { + void operator()( IStackAllocator* ) const; +}; + +} // namespace NeoML diff --git a/NeoMathEngine/src/MemoryEngineMixin.cpp b/NeoMathEngine/src/MemoryEngineMixin.cpp index 9b967980b..391a6bd34 100644 --- a/NeoMathEngine/src/MemoryEngineMixin.cpp +++ b/NeoMathEngine/src/MemoryEngineMixin.cpp @@ -19,11 +19,8 @@ limitations under the License. #include #include #include -#include -#include #include #include -#include namespace NeoML { @@ -32,9 +29,9 @@ void CMemoryEngineMixin::InitializeMemory( IRawMemoryManager* _rawManager, size_ { MemoryAlignment = _memoryAlignment; MemoryPool.reset( new CMemoryPool( _memoryLimit == 0 ? SIZE_MAX : _memoryLimit, _rawManager, _reuse ) ); - DeviceStackAllocator.reset( new CDeviceStackAllocator( *MemoryPool, MemoryAlignment ) ); + DeviceStackAllocator.reset( CreateStackAllocator( TStackAlloc::Device, MemoryPool.get(), MemoryAlignment ) ); if( _hostStack == true ) { - HostStackAllocator.reset( new CHostStackAllocator( MemoryAlignment ) ); + HostStackAllocator.reset( CreateStackAllocator( TStackAlloc::Host, /*memoryPool*/nullptr, MemoryAlignment ) ); } } @@ -191,7 +188,7 @@ void* CMemoryEngineMixin::GetBuffer( const CMemoryHandle& handle, size_t pos, si ASSERT_EXPR( handle.GetMathEngine() == this ); const size_t realSize = size + bufferHeaderSize; - char* result = static_cast( HostStackAllocator->Alloc( realSize ) ); + char* result = static_cast( static_cast( HostStackAllocator->Alloc( realSize ) ) ); size_t* header = reinterpret_cast( result ); header[0] = pos; diff --git a/NeoMathEngine/src/MemoryEngineMixin.h b/NeoMathEngine/src/MemoryEngineMixin.h index bc4091363..3ba2ce67f 100644 --- a/NeoMathEngine/src/MemoryEngineMixin.h +++ b/NeoMathEngine/src/MemoryEngineMixin.h @@ -16,16 +16,14 @@ limitations under the License. #pragma once #include +#include #include +#include #include #include namespace NeoML { -class CDeviceStackAllocator; -class CHostStackAllocator; -class CMemoryPool; - // Memory management engine base class class CMemoryEngineMixin : public IMathEngine { public: @@ -57,8 +55,8 @@ class CMemoryEngineMixin : public IMathEngine { int MemoryAlignment = 0; // allocation alignment mutable std::mutex Mutex; // protecting the data below from non-thread-safe use std::unique_ptr MemoryPool; // memory manager - std::unique_ptr DeviceStackAllocator; // stack allocator for GPU memory - std::unique_ptr HostStackAllocator; // stack allocator for regular memory + std::unique_ptr DeviceStackAllocator; // stack allocator for GPU memory + std::unique_ptr HostStackAllocator; // stack allocator for regular memory void CleanUpSpecial() override {} }; diff --git a/NeoMathEngine/src/MemoryHandleInternal.h b/NeoMathEngine/src/MemoryHandleInternal.h index 73db2dcb0..7f02b99a2 100644 --- a/NeoMathEngine/src/MemoryHandleInternal.h +++ b/NeoMathEngine/src/MemoryHandleInternal.h @@ -1,4 +1,4 @@ -/* Copyright © 2017-2020 ABBYY Production LLC +/* Copyright © 2017-2024 ABBYY Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -16,12 +16,9 @@ limitations under the License. #pragma once #include - #ifdef NEOML_USE_VULKAN - #include - -#endif +#endif // NEOML_USE_VULKAN namespace NeoML { @@ -31,28 +28,28 @@ class CMemoryHandleInternal : public CCrtAllocatedObject { #ifdef NEOML_USE_VULKAN static CVulkanMemory* GetRawAllocation( const CMemoryHandle& handle ) - { return reinterpret_cast( const_cast( handle.object ) ); } -#endif + { return reinterpret_cast( const_cast( handle.Object ) ); } +#endif // NEOML_USE_VULKAN #ifdef NEOML_USE_METAL static void* GetRawAllocation( const CMemoryHandle& handle ) - { return const_cast( handle.object ); } -#endif + { return const_cast( handle.Object ); } +#endif // NEOML_USE_METAL #if (defined NEOML_USE_METAL) | (defined NEOML_USE_VULKAN) - static ptrdiff_t GetRawOffset( const CMemoryHandle& handle ) { return handle.offset; } -#endif + static ptrdiff_t GetRawOffset( const CMemoryHandle& handle ) { return handle.Offset; } +#endif // NEOML_USE_METAL || NEOML_USE_VULKAN template static Type* GetRaw( const CTypedMemoryHandle& handle ) - { return const_cast( reinterpret_cast( reinterpret_cast( handle.object ) + handle.offset ) ); } + { return const_cast( reinterpret_cast( reinterpret_cast( handle.Object ) + handle.Offset ) ); } template static const Type* GetRaw( const CTypedMemoryHandle& handle ) - { return reinterpret_cast( reinterpret_cast( handle.object ) + handle.offset ); } + { return reinterpret_cast( reinterpret_cast( handle.Object ) + handle.Offset ); } static void* GetRaw( const CMemoryHandle& handle ) - { return const_cast( reinterpret_cast( reinterpret_cast( handle.object ) + handle.offset ) ); } + { return const_cast( reinterpret_cast( reinterpret_cast( handle.Object ) + handle.Offset ) ); } static CMemoryHandle CreateMemoryHandle( IMathEngine* mathEngine, const void* object ) { return CMemoryHandle( mathEngine, object, 0 ); } }; @@ -62,21 +59,21 @@ inline static void* GetRawAllocation( const CMemoryHandle& handle ) { return CMemoryHandleInternal::GetRawAllocation( handle ); } -#endif +#endif // NEOML_USE_METAL #ifdef NEOML_USE_VULKAN inline static CVulkanMemory* GetRawAllocation( const CMemoryHandle& handle ) { return CMemoryHandleInternal::GetRawAllocation( handle ); } -#endif +#endif // NEOML_USE_VULKAN #if (defined NEOML_USE_METAL) | (defined NEOML_USE_VULKAN) inline static ptrdiff_t GetRawOffset( const CMemoryHandle& handle ) { return CMemoryHandleInternal::GetRawOffset( handle ); } -#endif +#endif // NEOML_USE_METAL || NEOML_USE_VULKAN template inline static Type* GetRaw( const CTypedMemoryHandle& handle ) diff --git a/NeoMathEngine/src/MemoryPool.h b/NeoMathEngine/src/MemoryPool.h index 9dfd83283..34505179c 100644 --- a/NeoMathEngine/src/MemoryPool.h +++ b/NeoMathEngine/src/MemoryPool.h @@ -16,8 +16,6 @@ limitations under the License. #pragma once #include -#include -#include #include #include #include