Gpu support add vectors

.clang-format

@@ -0,0 +1,81 @@
+---
+# Need clang-format version 3.8 or above
+
+Language:        Cpp
+
+AccessModifierOffset: -2
+AlignAfterOpenBracket: true
+AlignConsecutiveAssignments: false
+AlignConsecutiveDeclarations: false
+AlignEscapedNewlinesLeft: true
+AlignOperands:   true
+AlignTrailingComments: true
+AllowAllParametersOfDeclarationOnNextLine: true
+AllowShortBlocksOnASingleLine: false
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: Empty
+AllowShortIfStatementsOnASingleLine: false
+AllowShortLoopsOnASingleLine: false
+AlwaysBreakAfterReturnType: None
+AlwaysBreakBeforeMultilineStrings: true
+AlwaysBreakTemplateDeclarations: true
+BinPackArguments: true
+BinPackParameters: true
+BreakBeforeBinaryOperators: None
+BreakBeforeBraces: Custom
+BraceWrapping:
+  AfterClass:      false
+  AfterControlStatement: false
+  AfterEnum:       false
+  AfterFunction:   false
+  AfterNamespace:  false
+  AfterObjCDeclaration: false
+  AfterStruct:     false
+  AfterUnion:      false
+  BeforeCatch:     true
+  BeforeElse:      true
+  IndentBraces:    false
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializersBeforeComma: false
+ColumnLimit:     100
+CommentPragmas:  '^ IWYU pragma:'
+ConstructorInitializerAllOnOneLineOrOnePerLine: true
+ConstructorInitializerIndentWidth: 4
+ContinuationIndentWidth: 4
+Cpp11BracedListStyle: true
+DerivePointerAlignment: false
+DisableFormat:   false
+ForEachMacros:   [ foreach, Q_FOREACH, BOOST_FOREACH ]
+IndentCaseLabels: true
+IndentWidth:     2
+IndentWrappedFunctionNames: false
+KeepEmptyLinesAtTheStartOfBlocks: false
+MacroBlockBegin: ''
+MacroBlockEnd:   ''
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: None
+PenaltyBreakBeforeFirstCallParameter: 10
+PenaltyBreakComment: 300
+PenaltyBreakFirstLessLess: 400
+PenaltyBreakString: 10000
+PenaltyExcessCharacter: 10
+PenaltyReturnTypeOnItsOwnLine: 100000000
+PointerAlignment: Left
+#ReflowComments: true # Supported only from clang 3.9
+SortIncludes: false
+SortUsingDeclarations: false
+SpaceAfterCStyleCast: false
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeParens: ControlStatements
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 2
+SpacesInAngles:  false
+SpacesInContainerLiterals: true
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+Standard:        Cpp11
+TabWidth:        8
+UseTab:          Never
+...
+

CMake/mrpapp_cuda.cmake

@@ -1,27 +1,38 @@
-# // Copyright (C) 2023 UT-Battelle, LLC
+# // Copyright (C) 2024 UT-Battelle, LLC
 # // All rights reserved.
 # //
 # // See LICENSE for terms of usage.
 # //
 
 # Checks for CUDA and accordingly sets MRPAPP_HAVE_CUDA
 # In addition, set MRPAPP_GPU_LIBS.
+message("checking CUDA environment")
 set(CMAKE_CUDA_ARCHITECTURES "70" CACHE STRING "Name of the real architecture to build for.")
 
 set(MRPAPP_HAVE_CUDA FALSE CACHE INTERNAL "")
 set(MRPAPP_GPU_LIBS "" CACHE INTERNAL "")
 
-# Find CUDA.
+include(mrpapp_defines)
 include(CheckLanguage)
 
+if(NOT CMAKE_CUDA_FLAGS MATCHES "allow-unsupported-compiler")
+  set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} --allow-unsupported-compiler")
+endif()
+
+set(CMAKE_CUDA_HOST_COMPILER
+  ${CMAKE_CXX_COMPILER}
+  CACHE STRING "nvcc host compiler passed via -ccbin")
+
+# Find CUDA.
 find_package(CUDAToolkit REQUIRED)
 check_language(CUDA)
 if (CMAKE_CUDA_COMPILER)
+  message("Found CUDA compiler!")
   enable_language(CUDA)
   set(MRPAPP_HAVE_CUDA TRUE CACHE INTERNAL "")
   set(MRPAPP_HAVE_GPU TRUE CACHE INTERNAL "")
-  dca_add_haves_define(MRPAPP_HAVE_CUDA)
-  dca_add_haves_define(MRPAPP_HAVE_GPU)
+  mrpapp_add_define(MRPAPP_HAVE_CUDA)
+  mrpapp_add_define(MRPAPP_HAVE_GPU)
 
   list(APPEND MRPAPP_GPU_LIBS CUDA::cudart CUDA::cublas)
   set(MRPAPP_CUDA_PROPERTIES "CMAKE_CUDA_ARCHITECTURES 70")

CMake/mrpapp_defines.cmake

@@ -0,0 +1,39 @@
+# // Copyright (C) 2024 UT-Battelle, LLC
+# // All rights reserved.
+# //
+# // See LICENSE for terms of usage.
+# //
+
+################################################################################
+# Adds a definition to the global 'have definitions' string.
+function(mrpapp_add_define definition)
+  if(ARGN)
+    set_property(GLOBAL APPEND PROPERTY MRPAPP_CXX_DEFINITIONS "${definition} ${ARGN}")
+  else()
+    set_property(GLOBAL APPEND PROPERTY MRPAPP_CXX_DEFINITIONS "${definition}")
+  endif()
+endfunction()
+
+################################################################################
+# Generates in the build directory the haves_defines.hpp that contains all 'haves preprocessor
+# definitions'.
+function(mrpapp_write_definitions_file)
+  get_property(MRPAPP_CXX_DEFINITIONS_VAR GLOBAL PROPERTY MRPAPP_CXX_DEFINITIONS)
+
+  list(SORT MRPAPP_CXX_DEFINITIONS_VAR)
+  list(REMOVE_DUPLICATES MRPAPP_CXX_DEFINITIONS_VAR)
+  list(REMOVE_ITEM MRPAPP_CXX_DEFINITIONS_VAR "")
+
+  set(mrpapp_cxx_defines_string "")
+  message("cxx definitions: ${MRPAPP_CXX_DEFINITIONS_VAR}")
+  foreach(def ${MRPAPP_CXX_DEFINITIONS_VAR})
+    string(CONCAT mrpapp_cxx_defines_string
+        "${mrpapp_cxx_defines_string}"
+        "#ifndef ${def}\n"
+        " #define ${def} ${${def}_define}\n"
+        "#endif\n\n")
+  endforeach()
+
+  configure_file("${PROJECT_SOURCE_DIR}/defines.hpp.in"
+    "${CMAKE_BINARY_DIR}/defines.hpp")
+endfunction()

CMake/mrpapp_hip.cmake

@@ -1,6 +1,12 @@
 ################################################################################
 # Author: Peter Doak, doakpw@ornl.gov, Oak Ridge National Lab
 #
+# // Copyright (C) 2024 UT-Battelle, LLC
+# // All rights reserved.
+# //
+# // See LICENSE for terms of usage.
+# //
+#
 # Checks for HIP and and accordingly sets MRPAPP_HAVE_HIP
 
 set(ROCM_ROOT
@@ -53,6 +59,7 @@ set(MRPAPP_HAVE_HIP FALSE CACHE INTERNAL "")
 set(MRPAPP_HAVE_MAGMA FALSE CACHE INTERNAL "")
 set(MRPAPP_GPU_LIBS "" CACHE INTERNAL "")
 
+include(mrpapp_defines)
 include(CheckLanguage)
 check_language(HIP)
 if (CMAKE_HIP_COMPILER)
@@ -61,9 +68,9 @@ if (CMAKE_HIP_COMPILER)
   set(MRPAPP_HAVE_HIP TRUE CACHE INTERNAL "")
   set(MRPAPP_HAVE_GPU TRUE CACHE INTERNAL "")
   # Probably probably these should be public properties of the hip targets
-  dca_add_haves_define(MRPAPP_HAVE_HIP)
-  dca_add_haves_define(MRPAPP_HAVE_GPU)
-  dca_add_haves_define(__HIP_PLATFORM_AMD__)
+  mrpapp_add_define(MRPAPP_HAVE_HIP)
+  mrpapp_add_define(MRPAPP_HAVE_GPU)
+  mrpapp_add_define(__HIP_PLATFORM_AMD__)
   list(APPEND MRPAPP_GPU_LIBS hip::host roc::hipblas)
   set(MRPAPP_HIP_PROPERTIES "CMAKE_HIP_ARCHITECTURES gfx906,gfx908")
   set(CMAKE_HIP_STANDARD 17)

CMake/mrpapp_linking.cmake

@@ -0,0 +1,47 @@
+
+#link the correct gpu runtime library
+function(mrpapp_gpu_runtime_link target_name)
+  if(MRPAPP_HAVE_HIP)
+    target_link_libraries(${target_name} PUBLIC hip::host roc::hipblas roc::hipsparse)
+    message("linking target ${target_name} to hip::host")
+  elseif(MRPAPP_HAVE_CUDA)
+    target_link_libraries(${target_name} PUBLIC CUDA::cudart)
+  endif()
+endfunction()
+
+#link the correct gpu runtime library
+function(mrpapp_gpu_blas_link target_name)
+  if(MRPAPP_HAVE_HIP)
+    target_link_libraries(${target_name} PUBLIC roc::hipblas roc::hipsparse)
+    message("linking target ${target_name} to roc::hipblas")
+  elseif(MRPAPP_HAVE_CUDA)
+    target_link_libraries(${target_name} PUBLIC CUDA::cublas)
+  endif()
+endfunction()
+
+function(mrpapp_gpu_device_link target_name)
+  if(MRPAPP_HAVE_HIP)
+    set_target_properties( ${target_name} PROPERTIES LINKER_LANGUAGE "HIP")
+    set_target_properties( ${target_name}
+      PROPERTIES HIP_SEPARABLE_COMPILATION ON)
+    set_target_properties( ${target_name}
+      PROPERTIES HIP_RESOLVE_DEVICE_SYMBOLS ON)
+    target_link_libraries(${target_name} PRIVATE hip::device roc::hipblas roc::hipsparse roc::rocthrust)
+    get_target_property(_srcs ${target_name} SOURCES)
+    get_target_property(_src_dir ${target_name} SOURCE_DIR)
+    #
+    # Mark all cu source files as HIP code.
+    foreach(_src IN LISTS _srcs)
+        #message("${_src_dir}/${_src}")
+        if(_src MATCHES ".*\.cu$")
+            set_source_files_properties(${_src} PROPERTIES LANGUAGE HIP)
+        endif()
+    endforeach()
+  elseif(MRPAPP_HAVE_CUDA)
+    set_target_properties( ${target_name}
+      PROPERTIES CUDA_SEPARABLE_COMPILATION ON)
+    set_target_properties( ${target_name}
+      PROPERTIES CUDA_RESOLVE_DEVICE_SYMBOLS ON)
+   # target_compile_definitions(lapack_kernels PRIVATE MRPAPP_HAVE_CUDA)
+  endif()
+endfunction()

CMake/mrpapp_tests.cmake

@@ -0,0 +1,28 @@
+# Runs unit tests
+function(ADD_UNIT_TEST TESTNAME PROCS THREADS TEST_BINARY)
+  message(VERBOSE "Adding test ${TESTNAME}")
+  math(EXPR TOT_PROCS "${PROCS} * ${THREADS}")
+  if(HAVE_MPI)
+    add_test(NAME ${TESTNAME} COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${PROCS} ${MPIEXEC_PREFLAGS}
+                                      ${TEST_BINARY} ${ARGN})
+    set(TEST_ADDED TRUE)
+  else()
+    if((${PROCS} STREQUAL "1"))
+      add_test(NAME ${TESTNAME} COMMAND ${TEST_BINARY} ${ARGN})
+      set(TEST_ADDED TRUE)
+    else()
+      message(VERBOSE "Disabling test ${TESTNAME} (building without MPI)")
+    endif()
+  endif()
+
+  if(TEST_ADDED)
+    set_tests_properties(${TESTNAME} PROPERTIES PROCESSORS ${TOT_PROCS} ENVIRONMENT OMP_NUM_THREADS=${THREADS}
+                                                PROCESSOR_AFFINITY TRUE)
+
+    if(ENABLE_GPU)
+      set_tests_properties(${TESTNAME} PROPERTIES RESOURCE_LOCK exclusively_owned_gpus)
+    endif()
+
+  endif()
+
+endfunction()

CMakeLists.txt

@@ -1,3 +1,9 @@
+# // Copyright (C) 2024 UT-Battelle, LLC
+# // All rights reserved.
+# //
+# // See LICENSE for terms of usage.
+# //
+
 ######################################################################
 # CMake version and policies
 ######################################################################
@@ -19,10 +25,10 @@ set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
 set(PROJECT_CMAKE ${mrpapp_SOURCE_DIR}/CMake)
 list(APPEND CMAKE_MODULE_PATH ${PROJECT_CMAKE})
 
-set(QMC_CXX_STANDARD
+set(MRPAPP_CXX_STANDARD
     17
     CACHE STRING "MRPAPP C++ language standard")
-if(NOT QMC_CXX_STANDARD EQUAL 17)
+if(NOT MRPAPP_CXX_STANDARD EQUAL 17)
   message(WARNING "C++17 is the only language standard officially supported by this MRPAPP version. "
                   "Using other versions of the C++ standard is unsupported and done entirely at user's own risk.")
 endif()
@@ -35,7 +41,7 @@ endif(USE_MPI)
 option(ENABLE_CUDA "Build with GPU support through CUDA" OFF)
 option(ENABLE_HIP  "Build with GPU support through HIP" OFF)
 set(ENABLE_GPU "$<IF:$<OR:$<BOOL:${ENABLE_CUDA}>,$<BOOL:${ENABLE_CUDA}>>,ON,OFF>")
-
+message ("enable gpu: ${ENABLE_GPU}")
 set(MRPAPP_GPU_LIBS "" CACHE INTERNAL "")
 
 if(ENABLE_GPU)
@@ -45,22 +51,48 @@ if(ENABLE_GPU)
   if(ENABLE_HIP)
     include(mrapp_hip)
   endif(ENABLE_HIP)
-
   if(MRPAPP_HAVE_CUDA OR MRPAPP_HAVE_HIP)
     include(DetermineDeviceArchitectures)
-    message(STATUS "GPU device architectures: ${QMC_GPU_ARCHS}")
+    message(STATUS "GPU device architectures: ${MRPAPP_GPU_ARCHS}")
   endif()
 endif(ENABLE_GPU)
 
 find_package(LAPACK REQUIRED)
 find_package(BLAS REQUIRED)
 
+include(ExternalProject)
+find_package(Git REQUIRED)
+
+find_package(Catch2 3)
+if (NOT CATCH2_FOUND)
+ExternalProject_Add(
+  catch2
+  PREFIX ${CMAKE_BINARY_DIR}/catch2
+  GIT_REPOSITORY git@github.com:catchorg/Catch2.git
+  GIT_TAG devel
+  TIMEOUT 30
+  UPDATE_COMMAND ${GIT_EXECUTABLE} pull
+  CMAKE_ARGS "-DCMAKE_INSTALL_PREFIX=${CMAKE_BINARY_DIR}/catch2"
+  LOG_DOWNLOAD ON)
+
+set(CMAKE_PREFIX_PATH ${CMAKE_BINARY_DIR}/catch2)
+find_package(Catch2 3 REQUIRED)
+endif()
+
 set(MRPAPP_SRC
-  main.cpp)
+main.cpp)
 
 set(MRPAPP_MODEL "1BAND" CACHE STRING "RPA model options")
 set_property(CACHE MRPAPP_MODEL PROPERTY STRINGS SRRUO SRRUO3D SRRUO3DSUH 1BAND 1BANDWSPIN BILAYER_FESC BILAYER_1BAND ORTHOIIBILAYER BSCCOBILAYER BILAYER_FESC BAFEAS KFE2SE2 FOURORBITAL TBFILE COUPLEDLADDERS NDNIO2 MODELFROMFILESO KAGOME 1BANDABWSPIN 1BANDALTERMAGNET 1BANDAB)
 
+include(mrpapp_defines)
+mrpapp_write_definitions_file()
+
+include(mrpapp_linking)
+
+add_subdirectory(platform)
+add_subdirectory(linalg)
+
 add_executable(mrpapp ${MRPAPP_SRC})
 target_include_directories(mrpapp PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}"
   ${PROJECT_SOURCE_DIR}/PartialPsimag 
@@ -74,4 +106,7 @@ set(mpi_targs MPI::MPI_C MPI::MPI_CXX)
 set(MPI_TARGETS "$<$<BOOL:${MPI_FOUND}>:${mpi_targs}>")
 target_link_libraries(mrpapp ${MRPAPP_GPU_LIBS} LAPACK::LAPACK BLAS::BLAS ${MPI_TARGETS} )
 
-add_custom_target(genexdebug COMMAND ${CMAKE_COMMAND} -E echo "$<$<BOOL:${MPI_FOUND}>:USE_MPI>")
+add_custom_target(genexdebug COMMAND ${CMAKE_COMMAND} -E echo "$<IF:$<OR:$<BOOL:${ENABLE_CUDA}>,$<BOOL:${ENABLE_CUDA}>>,ON,OFF>")
+#$<$<BOOL:${MPI_FOUND}>:USE_MPI>")
+
+include(mrpapp_tests)

defines.hpp.in

@@ -0,0 +1,12 @@
+# // Copyright (C) 2023 UT-Battelle, LLC
+# // All rights reserved.
+# //
+# // See LICENSE for terms of usage.
+# //
+
+#ifndef MRPAPP_DEFINES_HPP
+#define MRPAPP_DEFINES_HPP
+
+@mrpapp_cxx_defines_string@
+
+#endif    

gaps3D.h

@@ -98,9 +98,9 @@ class gap3D { // simple s+- for 5-orbital 1111 model 2D
                                        // description
       param.parity = 1.0;              // even parity (d-wave) gap
       if (band == 0) {                 // bonding band
-        return param.Delta0 * (abs(cos(k[0])) - cos(k[1]));
+        return param.Delta0 * (std::abs(cos(k[0])) - cos(k[1]));
       } else { // antibonding band (shift kx by pi)
-        return param.Delta0 * (-abs(cos(k[0])) - cos(k[1]));
+        return param.Delta0 * (-std::abs(cos(k[0])) - cos(k[1]));
       }
     } else if (param.gAmpl == "SrRuO_helical" ||
                param.gAmpl == "SrRuO_chiral" ||