From 85718893d2510c8a2e8c8e94c768ce6a67f94703 Mon Sep 17 00:00:00 2001
From: Jon Rood <jon.rood@nrel.gov>
Date: Mon, 16 Oct 2023 16:18:51 -0600
Subject: [PATCH] Add HIP init before MPI init for MPI issue workaround on
 Frontier. (#62)

---
 app/exawind/CMakeLists.txt | 4 ++++
 app/exawind/exawind.cpp    | 9 +++++++++
 2 files changed, 13 insertions(+)

diff --git a/app/exawind/CMakeLists.txt b/app/exawind/CMakeLists.txt
index d5d4f64..99f16f6 100644
--- a/app/exawind/CMakeLists.txt
+++ b/app/exawind/CMakeLists.txt
@@ -7,6 +7,10 @@ target_link_libraries(${EXAWIND_EXE_NAME} PRIVATE
 
 target_include_directories(${EXAWIND_EXE_NAME} SYSTEM PRIVATE ${YAML_CPP_INCLUDE_DIR})
 
+if(EXAWIND_ENABLE_ROCM)
+  target_compile_definitions(${EXAWIND_EXE_NAME} PRIVATE EXAWIND_ENABLE_ROCM)
+endif()
+
 if(CXX_FILESYSTEM_HAVE_FS)
   target_compile_definitions(${EXAWIND_EXE_NAME} PRIVATE EXAWIND_HAS_STD_FILESYSTEM)
   target_link_libraries(${EXAWIND_EXE_NAME} PRIVATE
diff --git a/app/exawind/exawind.cpp b/app/exawind/exawind.cpp
index c4cd405..eeaf92f 100644
--- a/app/exawind/exawind.cpp
+++ b/app/exawind/exawind.cpp
@@ -11,6 +11,11 @@
 #include "yaml-cpp/yaml.h"
 #include "tioga.h"
 
+// Workaround for MPI issue on OLCF Frontier machine
+#ifdef EXAWIND_ENABLE_ROCM
+#include <hip/hip_runtime.h>
+#endif
+
 static std::string usage(std::string name)
 {
     return "usage: " + name +
@@ -36,6 +41,10 @@ replace_extension(const std::string& filepath, const std::string& newExt)
 
 int main(int argc, char** argv)
 {
+// Workaround for MPI issue on OLCF Frontier machine
+#ifdef EXAWIND_ENABLE_ROCM
+    hipInit(0);
+#endif
     MPI_Init(&argc, &argv);
     int psize, prank;
     MPI_Comm_size(MPI_COMM_WORLD, &psize);