Use custom coalescing heuristic and introduce a number of new configu…

…ration options for memory pools (#1183)

include/dlaf/init.h

@@ -37,10 +37,18 @@ struct configuration {
   // - getOptionsDescription to add a corresponding command line option
   std::size_t num_np_gpu_streams_per_thread = 3;
   std::size_t num_hp_gpu_streams_per_thread = 3;
+  std::size_t umpire_host_memory_pool_initial_block_bytes = 1 << 30;
+  std::size_t umpire_host_memory_pool_next_block_bytes = 1 << 30;
+  std::size_t umpire_host_memory_pool_alignment_bytes = 16;
+  double umpire_host_memory_pool_coalescing_free_ratio = 1.0;
+  double umpire_host_memory_pool_coalescing_reallocation_ratio = 1.0;
+  std::size_t umpire_device_memory_pool_initial_block_bytes = 1 << 30;
+  std::size_t umpire_device_memory_pool_next_block_bytes = 1 << 30;
+  std::size_t umpire_device_memory_pool_alignment_bytes = 16;
+  double umpire_device_memory_pool_coalescing_free_ratio = 1.0;
+  double umpire_device_memory_pool_coalescing_reallocation_ratio = 1.0;
   std::size_t num_gpu_blas_handles = 16;
   std::size_t num_gpu_lapack_handles = 16;
-  std::size_t umpire_host_memory_pool_initial_bytes = 1 << 30;
-  std::size_t umpire_device_memory_pool_initial_bytes = 1 << 30;
   std::string mpi_pool = "mpi";
 };
 

include/dlaf/memory/memory_chunk.h

@@ -27,11 +27,15 @@ namespace memory {
 
 namespace internal {
 umpire::Allocator& getUmpireHostAllocator();
-void initializeUmpireHostAllocator(std::size_t initial_bytes);
+void initializeUmpireHostAllocator(std::size_t initial_block_bytes, std::size_t next_block_bytes,
+                                   std::size_t alignment_bytes, double coalesce_free_ratio,
+                                   double coalesce_reallocation_ratio);
 void finalizeUmpireHostAllocator();
 
 #ifdef DLAF_WITH_GPU
-void initializeUmpireDeviceAllocator(std::size_t initial_bytes);
+void initializeUmpireDeviceAllocator(std::size_t initial_block_bytes, std::size_t next_block_bytes,
+                                     std::size_t alignment_bytes, double coalesce_free_ratio,
+                                     double coalesce_reallocation_ratio);
 void finalizeUmpireDeviceAllocator();
 umpire::Allocator& getUmpireDeviceAllocator();
 #endif