Added GlobalDataInterface

CMakeLists.txt

@@ -26,7 +26,7 @@ option(PCMS_ENABLE_OMEGA_H "enable Omega_h field adapter" OFF)
 option(PCMS_ENABLE_C "Enable pcms C api" ON)
 
 option(PCMS_ENABLE_PRINT "PCMS print statements enabled" ON)
-option(PCMS_ENABLE_SPDLOG "use spdlog for logging" ON)
+option(PCMS_ENABLE_SPDLOG "use spdlog for logging" OFF)
 
 if(PCMS_ENABLE_SPDLOG)
   find_package(spdlog REQUIRED)

examples/external-usage-example/main.cpp

@@ -1,5 +1,6 @@
 #include <iostream>
 #include <pcms/utility/print.h>
+#include <pcms/create_field.h>
 
 int main()
 {

src/CMakeLists.txt

@@ -1,3 +1,5 @@
+
+
 # TODO split out the field transfer library
 set(
   PCMS_HEADERS
@@ -9,52 +11,51 @@ set(
   pcms/field.h
   pcms/create_field.h
   pcms/field_communicator.h
+  pcms/global_communicator.h
   pcms/field_communicator2.h
   pcms/field_evaluation_methods.h
   pcms/partition.h
   pcms/coupler.h
   pcms/coordinate_system.h
   pcms/field_layout.h
-  pcms/adapter/point_cloud/point_cloud_layout.h
-  pcms/adapter/point_cloud/point_cloud.h
-  pcms/adapter/omega_h/omega_h_field_layout.h
-  pcms/adapter/omega_h/omega_h_field2.h
 )
 
 set(
   PCMS_SOURCES
   pcms.cpp
   pcms/create_field.cpp
-  pcms/adapter/xgc/xgc_field_adapter.h
   pcms/adapter/point_cloud/point_cloud_layout.cpp
   pcms/adapter/point_cloud/point_cloud.cpp
   pcms/adapter/omega_h/omega_h_field_layout.cpp
   pcms/adapter/omega_h/omega_h_field2.cpp
-  pcms/adapter/xgc/xgc_field_adapter.h
 )
-
+set(
+  ADAPTER_HEADERS
+  ${CMAKE_SOURCE_DIR}/src/pcms/adapter/point_cloud
+)
 configure_file(pcms/version.h.in pcms/version.h)
 configure_file(pcms/configuration.h.in pcms/configuration.h)
 list(APPEND PCMS_HEADERS ${CMAKE_CURRENT_BINARY_DIR}/pcms/version.h ${CMAKE_CURRENT_BINARY_DIR}/pcms/configuration.h)
 
 add_subdirectory(pcms/utility)
 
-
 if(PCMS_ENABLE_XGC)
   list(APPEND PCMS_SOURCES pcms/adapter/xgc/xgc_reverse_classification.cpp)
-  list(APPEND PCMS_HEADERS pcms/adapter/xgc/xgc_reverse_classification.h)
+  list(APPEND ADAPTER_HEADERS ${CMAKE_SOURCE_DIR}/src/pcms/adapter/xgc)
 endif()
 if(PCMS_ENABLE_OMEGA_H)
   list(APPEND PCMS_SOURCES pcms/point_search.cpp)
   list(
     APPEND
     PCMS_HEADERS
-    pcms/adapter/omega_h/omega_h_field.h
     pcms/transfer_field.h
     pcms/transfer_field2.h
     pcms/uniform_grid.h
     pcms/point_search.h
+    #install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/pcms/adapter/omega_h/omega_h_field.h
+    #DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/pcms/adapter/omega_h)
   )
+  list(APPEND ADAPTER_HEADERS ${CMAKE_SOURCE_DIR}/src/pcms/adapter/omega_h)
 endif()
 
 find_package(Kokkos REQUIRED)
@@ -147,6 +148,14 @@ install(
   DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
   PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/pcms
 )
+
+# Mirror the src tree for adapter headers
+foreach(hdr IN LISTS ADAPTER_HEADERS)
+  install(DIRECTORY ${hdr}
+          DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/pcms/adapter
+          FILES_MATCHING PATTERN "*.h")
+endforeach ()
+
 # install external headers
 install(
   DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/pcms/external/

src/pcms/coupler.h

@@ -1,13 +1,14 @@
 #ifndef PCMS_COUPLER_H
 #define PCMS_COUPLER_H
+#include "global_communicator.h"
 #include "pcms/utility/common.h"
 #include "pcms/field_communicator.h"
 #include "pcms/adapter/omega_h/omega_h_field.h"
 #include "pcms/utility/profile.h"
 
+
 namespace pcms
 {
-
 // to avoid having any redev:: types in the user interface
 using ProcessType = redev::ProcessType;
 
@@ -115,7 +116,32 @@ class CoupledField
 private:
   std::unique_ptr<CoupledFieldConcept> coupled_field_;
 };
+template <typename T>
+class GlobalDataInterface
+{
+  public:
+  GlobalDataInterface( const std::string& name , MPI_Comm mpi_comm, redev::Channel& channel)
+    : mpi_comm_(mpi_comm), comm_(GlobalCommunicator<T>(name, mpi_comm_, channel)),
+      type_info_(typeid(T))
+  {
+    PCMS_FUNCTION_TIMER;
+  }
+  void Send(T* msg, std::string VarName, size_t msg_size, Mode mode = Mode::Synchronous)
+  {
+    PCMS_FUNCTION_TIMER;
+    comm_.Send(msg, VarName, msg_size, mode);
+  }
+  std::vector<T> Receive(std::string VarName, size_t msg_size, Mode mode = Mode::Synchronous)
+  {
+    PCMS_FUNCTION_TIMER;
+    return comm_.Receive(VarName, msg_size, mode);
+  }
+private:
+  MPI_Comm mpi_comm_;
+  const std::type_info& type_info_;
+  GlobalCommunicator<T> comm_;
 
+};
 class Application
 {
 public:
@@ -145,6 +171,12 @@ class Application
     }
     return &(it->second);
   }
+  template <typename T>
+  std::unique_ptr<GlobalDataInterface<T>> Add_GDI(std::string name, MPI_Comm mpi_comm)
+  {
+    PCMS_FUNCTION_TIMER;
+    return  std::make_unique<GlobalDataInterface<T>>(name, mpi_comm, channel_); // Use the existing applivatiocation channel
+  }
   void SendField(const std::string& name, Mode mode = Mode::Synchronous)
   {
     PCMS_FUNCTION_TIMER;

src/pcms/create_field.cpp

@@ -9,7 +9,7 @@ namespace pcms
 
 std::unique_ptr<FieldLayout> CreateLagrangeLayout(
   Omega_h::Mesh& mesh, int order, int num_components,
-  CoordinateSystem coordinate_system)
+  CoordinateSystem coordinate_system, std::string global_id_name)
 {
 
   std::array<int, 4> nodes_per_dim;
@@ -21,7 +21,7 @@ std::unique_ptr<FieldLayout> CreateLagrangeLayout(
   }
 
   return std::make_unique<OmegaHFieldLayout>(mesh, nodes_per_dim,
-                                             num_components, coordinate_system);
+                                             num_components, coordinate_system, global_id_name);
 }
 
 } // namespace pcms

src/pcms/create_field.h

@@ -13,7 +13,8 @@ namespace pcms
 {
 std::unique_ptr<FieldLayout> CreateLagrangeLayout(
   Omega_h::Mesh& mesh, int order, int num_components,
-  CoordinateSystem coordinate_system);
+  CoordinateSystem coordinate_system,
+  std::string global_id_name="global");
 } // namespace pcms
 
 #endif // CREATE_FIELD_H_

src/pcms/global_communicator.h

@@ -0,0 +1,49 @@
+#ifndef PCMS_GLOBAL_COMMUNICATOR_H
+#define PCMS_GLOBAL_COMMUNICATOR_H
+#endif // PCMS_GLOBAL_COMMUNICATOR_H
+
+#include <redev.h>
+
+namespace pcms
+{
+  using redev::Mode;
+  template <typename T>
+  struct GlobalCommunicator
+  {
+    using value_type = T;
+    public:
+    GlobalCommunicator(std::string name, MPI_Comm mpi_comm, redev::Channel& channel)
+      : mpi_comm(mpi_comm),
+        channel_(channel),
+        name_(std::move(name))
+    {
+      PCMS_FUNCTION_TIMER;
+      comm_ = channel_.CreateComm<T>(name_, mpi_comm, redev::CommType::Global );
+    }
+    GlobalCommunicator(const GlobalCommunicator&) = delete;
+    GlobalCommunicator& operator=(const GlobalCommunicator&) = delete;
+    GlobalCommunicator(GlobalCommunicator&&)= default;
+    GlobalCommunicator& operator=(GlobalCommunicator&&) = default;
+
+    void Send(T* msg, std::string VarName, size_t msg_size, Mode mode = Mode::Synchronous)
+    {
+      PCMS_FUNCTION_TIMER;
+      PCMS_ALWAYS_ASSERT(channel_.InSendCommunicationPhase());
+      comm_.SetCommParams( VarName, msg_size);
+      comm_.Send(msg, mode);
+    }
+    std::vector<T> Receive(std::string VarName, size_t msg_size, Mode mode = Mode::Synchronous)
+    {
+      PCMS_FUNCTION_TIMER;
+      PCMS_ALWAYS_ASSERT(channel_.InReceiveCommunicationPhase());
+      comm_.SetCommParams(VarName, msg_size);
+      auto data = comm_.Recv(mode);
+      return data;
+    }
+  private:
+    MPI_Comm mpi_comm;
+    redev::Channel& channel_;
+    std::string name_;
+    redev::BidirectionalComm<T> comm_;
+  };
+}

test/CMakeLists.txt

@@ -96,7 +96,37 @@ if(PCMS_ENABLE_OMEGA_H)
       ${d3d16p}
       ignored)
   endif()
-
+  add_exe(test_GDI)
+  tri_mpi_test(
+          TESTNAME
+          test_GDI
+          TIMEOUT
+          20
+          NAME1
+          app
+          EXE1
+          ./test_GDI
+          PROCS1
+          1
+          ARGS1
+          1
+          NAME2
+          rdv
+          EXE2
+          ./test_GDI
+          PROCS2
+          1
+          ARGS2
+          -1
+          NAME3
+          app
+          EXE3
+          ./test_GDI
+          PROCS3
+          1
+          ARGS3
+          0
+  )
   set(d3d8p ${PCMS_TEST_DATA_DIR}/d3d/d3d-full_9k_sfc_p8.osh/)
   add_exe(test_twoClientOverlap)
   if(HOST_NPROC GREATER_EQUAL 28)