Add HIP init before MPI init for MPI issue workaround on Frontier. (#62)

Exawind · Oct 16, 2023 · 8571889 · 8571889
1 parent d3c1aa4
commit 8571889
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 0 deletions.
diff --git a/app/exawind/CMakeLists.txt b/app/exawind/CMakeLists.txt
@@ -7,6 +7,10 @@ target_link_libraries(${EXAWIND_EXE_NAME} PRIVATE
 
 target_include_directories(${EXAWIND_EXE_NAME} SYSTEM PRIVATE ${YAML_CPP_INCLUDE_DIR})
 
+if(EXAWIND_ENABLE_ROCM)
+  target_compile_definitions(${EXAWIND_EXE_NAME} PRIVATE EXAWIND_ENABLE_ROCM)
+endif()
+
 if(CXX_FILESYSTEM_HAVE_FS)
   target_compile_definitions(${EXAWIND_EXE_NAME} PRIVATE EXAWIND_HAS_STD_FILESYSTEM)
   target_link_libraries(${EXAWIND_EXE_NAME} PRIVATE

diff --git a/app/exawind/exawind.cpp b/app/exawind/exawind.cpp
@@ -11,6 +11,11 @@
 #include "yaml-cpp/yaml.h"
 #include "tioga.h"
 
+// Workaround for MPI issue on OLCF Frontier machine
+#ifdef EXAWIND_ENABLE_ROCM
+#include <hip/hip_runtime.h>
+#endif
+
 static std::string usage(std::string name)
 {
     return "usage: " + name +
@@ -36,6 +41,10 @@ replace_extension(const std::string& filepath, const std::string& newExt)
 
 int main(int argc, char** argv)
 {
+// Workaround for MPI issue on OLCF Frontier machine
+#ifdef EXAWIND_ENABLE_ROCM
+    hipInit(0);
+#endif
     MPI_Init(&argc, &argv);
     int psize, prank;
     MPI_Comm_size(MPI_COMM_WORLD, &psize);