Wrappers for iPic3D

mpi-proxy-split/mpi-wrappers/mpi_file_wrappers.cpp

@@ -46,15 +46,63 @@ USER_DEFINED_WRAPPER(int, File_open, (MPI_Comm) comm, (const char *) filename,
   return retval;
 }
 
+USER_DEFINED_WRAPPER(int, File_set_view, (MPI_File) fh, (MPI_Offset) disp,
+                    (MPI_Datatype) etype, (MPI_Datatype) filetype,
+                    (const char*) datarep, (MPI_Info) info)
+{
+  int retval;
+  DMTCP_PLUGIN_DISABLE_CKPT();
+  MPI_Datatype real_etype = VIRTUAL_TO_REAL_TYPE(etype);
+  MPI_Datatype real_filetype = VIRTUAL_TO_REAL_TYPE(filetype);
+  JUMP_TO_LOWER_HALF(lh_info.fsaddr);
+  retval = NEXT_FUNC(File_set_view)(fh, disp, real_etype, real_filetype, datarep, info);
+  RETURN_TO_UPPER_HALF();
+  DMTCP_PLUGIN_ENABLE_CKPT();
+  return retval;
+}
+
+USER_DEFINED_WRAPPER(int, File_write, (MPI_File) fh, (const void *) buf, (int) count,
+                     (MPI_Datatype) datatype, (MPI_Status *) status)
+{
+  int retval;
+  DMTCP_PLUGIN_DISABLE_CKPT();
+  MPI_Datatype real_datatype = VIRTUAL_TO_REAL_TYPE(datatype);
+  JUMP_TO_LOWER_HALF(lh_info.fsaddr);
+  retval = NEXT_FUNC(File_write)(fh, buf, count, real_datatype, status);
+  RETURN_TO_UPPER_HALF();
+  DMTCP_PLUGIN_ENABLE_CKPT();
+  return retval;
+}
+
+USER_DEFINED_WRAPPER(int, File_write_all, (MPI_File) fh, (const void *) buf, (int) count,
+                     (MPI_Datatype) datatype, (MPI_Status *) status)
+{
+  int retval;
+  DMTCP_PLUGIN_DISABLE_CKPT();
+  MPI_Datatype real_datatype = VIRTUAL_TO_REAL_TYPE(datatype);
+  JUMP_TO_LOWER_HALF(lh_info.fsaddr);
+  retval = NEXT_FUNC(File_write_all)(fh, buf, count, real_datatype, status);
+  RETURN_TO_UPPER_HALF();
+  DMTCP_PLUGIN_ENABLE_CKPT();
+  return retval;
+}
+
+
+
 DEFINE_FNC(int, File_close, (MPI_File *) fh);
 DEFINE_FNC(int, File_get_size, (MPI_File) fh, (MPI_Offset *) size);
 DEFINE_FNC(int, File_read_at, (MPI_File) fh, (MPI_Offset) offset, (void *) buf,
            (int) count, (MPI_Datatype) datatype, (MPI_Status *) status);
-
+ 
 PMPI_IMPL(int, MPI_File_open, MPI_Comm comm, const char *filename, int amode,
           MPI_Info info, MPI_File *fh)
 PMPI_IMPL(int, MPI_File_close, MPI_File *fh)
 PMPI_IMPL(int, MPI_File_get_size, MPI_File fh, MPI_Offset *size)
 PMPI_IMPL(int, MPI_File_read_at, MPI_File fh, MPI_Offset offset, void *buf,
           int count, MPI_Datatype datatype, MPI_Status *status)
-
+PMPI_IMPL(int, MPI_File_write, MPI_File fh, const void *buf, int count,                
+           MPI_Datatype datatype, MPI_Status *status)
+PMPI_IMPL(int, MPI_File_set_view, MPI_File fh, MPI_Offset disp, MPI_Datatype etype,
+                     MPI_Datatype filetype, const char* datarep, MPI_Info info);
+PMPI_IMPL(int, MPI_File_write_all, MPI_File fh, const void *buf, int count,
+                       MPI_Datatype datatype, MPI_Status *status)

mpi-proxy-split/mpi-wrappers/mpi_request_wrappers.cpp

@@ -338,6 +338,9 @@ USER_DEFINED_WRAPPER(int, Wait, (MPI_Request*) request, (MPI_Status*) status)
       fflush(stdout);
 #endif
     }
+    if (flag) {
+      statusPtr->MPI_ERROR = MPI_SUCCESS;
+    }
     if (p2p_deterministic_skip_save_request == 0) {
       if (flag) LOG_POST_Wait(request, statusPtr);
     }
@@ -393,6 +396,38 @@ USER_DEFINED_WRAPPER(int, Request_get_status, (MPI_Request) request,
   return retval;
 }
 
+USER_DEFINED_WRAPPER(int, Cancel, (MPI_Request *) request)
+{
+  int retval;
+  DMTCP_PLUGIN_DISABLE_CKPT();
+  MPI_Request realRequest = VIRTUAL_TO_REAL_REQUEST(*request);
+  JUMP_TO_LOWER_HALF(lh_info.fsaddr);
+  if( realRequest == MPI_REQUEST_NULL ){
+    retval = MPI_SUCCESS;
+  } else {
+    retval = NEXT_FUNC(Cancel)(&realRequest);
+  }
+  RETURN_TO_UPPER_HALF();
+  DMTCP_PLUGIN_ENABLE_CKPT();
+  return retval;
+}
+
+USER_DEFINED_WRAPPER(int, Request_free, (MPI_Request *) request)
+{
+  int retval;
+  DMTCP_PLUGIN_DISABLE_CKPT();
+  MPI_Request realRequest = VIRTUAL_TO_REAL_REQUEST(*request);
+  JUMP_TO_LOWER_HALF(lh_info.fsaddr);
+  if( realRequest == MPI_REQUEST_NULL ){
+    retval = MPI_SUCCESS;
+  } else {
+    retval = NEXT_FUNC(Request_free)(&realRequest);
+  }
+  RETURN_TO_UPPER_HALF();
+  DMTCP_PLUGIN_ENABLE_CKPT();
+  return retval;
+}
+
 DEFINE_FNC(int, Get_elements, (const MPI_Status *) status,
            (MPI_Datatype) datatype, (int *) count);
 DEFINE_FNC(int, Get_elements_x, (const MPI_Status *) status,
@@ -418,4 +453,5 @@ PMPI_IMPL(int, MPI_Get_elements_x, const MPI_Status *status,
           MPI_Datatype datatype, MPI_Count *count)
 PMPI_IMPL(int, MPI_Request_get_status, MPI_Request request, int* flag,
           MPI_Status *status)
-
+PMPI_IMPL(int, MPI_Cancel, MPI_Request *request)
+PMPI_IMPL(int, MPI_Request_free, MPI_Request *request)

mpi-proxy-split/mpi-wrappers/mpi_type_wrappers.cpp

@@ -170,6 +170,31 @@ USER_DEFINED_WRAPPER(int, Type_create_struct, (int) count,
   return retval;
 }
 
+USER_DEFINED_WRAPPER(int, Type_create_subarray, (int) ndims, (const int*) array_of_sizes,
+                     (const int*) array_of_subsizes, (const int*) array_of_starts,
+                     (int) order, (MPI_Datatype) oldtype, (MPI_Datatype*) newtype)
+{
+  int retval;
+  DMTCP_PLUGIN_DISABLE_CKPT();
+  MPI_Datatype realType = VIRTUAL_TO_REAL_TYPE(oldtype);
+  JUMP_TO_LOWER_HALF(lh_info.fsaddr);
+  retval = NEXT_FUNC(Type_create_subarray)(ndims, array_of_sizes, array_of_subsizes,
+                                           array_of_starts,  order, realType, newtype);
+  RETURN_TO_UPPER_HALF();
+  if (retval == MPI_SUCCESS && MPI_LOGGING()) {
+    MPI_Datatype virtType = ADD_NEW_TYPE(*newtype);
+    *newtype = virtType;
+    FncArg sizes = CREATE_LOG_BUF(array_of_sizes, ndims * sizeof(int));
+    FncArg subsizes = CREATE_LOG_BUF(array_of_subsizes, ndims * sizeof(int));
+    FncArg starts = CREATE_LOG_BUF(array_of_starts, ndims * sizeof(int));
+    LOG_CALL(restoreTypes, Type_create_subarray, ndims, sizes, subsizes, starts,
+             order, oldtype, virtType);
+  }
+  DMTCP_PLUGIN_ENABLE_CKPT();
+  return retval;
+}
+
+
 #ifdef CRAY_MPICH_VERSION
 USER_DEFINED_WRAPPER(int, Type_struct, (int) count,
                      (const int*) array_of_blocklengths,
@@ -269,6 +294,9 @@ PMPI_IMPL(int, MPI_Type_hvector, int count, int blocklength,
 PMPI_IMPL(int, MPI_Type_create_struct, int count, const int array_of_blocklengths[],
           const MPI_Aint array_of_displacements[], const MPI_Datatype array_of_types[],
           MPI_Datatype *newtype)
+PMPI_IMPL(int, MPI_Type_create_subarray, int ndims, const int array_of_sizes[],
+                     const int array_of_subsizes[], const int array_of_starts[],
+                     int order, MPI_Datatype oldtype, MPI_Datatype *newtype)
 
 #ifdef CRAY_MPICH_VERSION
 PMPI_IMPL(int, MPI_Type_struct, int count, const int array_of_blocklengths[],

mpi-proxy-split/mpi-wrappers/mpi_unimplemented_wrappers.txt

@@ -48,7 +48,6 @@ int MPI_Bsend_init(const void *buf, int count, MPI_Datatype datatype,
                             MPI_Request *request);
 int MPI_Buffer_attach(void *buffer, int size);
 int MPI_Buffer_detach(void *buffer, int *size);
-int MPI_Cancel(MPI_Request *request);
 int MPI_Close_port(const char *port_name);
 int MPI_Comm_accept(const char *port_name, MPI_Info info, int root, MPI_Comm comm, MPI_Comm *newcomm);
 
@@ -97,7 +96,6 @@ int MPI_File_get_group(MPI_File fh, MPI_Group *group);
 int MPI_File_get_amode(MPI_File fh, int *amode);
 int MPI_File_set_info(MPI_File fh, MPI_Info info);
 int MPI_File_get_info(MPI_File fh, MPI_Info *info_used);
-int MPI_File_set_view(MPI_File fh, MPI_Offset disp, MPI_Datatype etype, MPI_Datatype filetype, const char *datarep, MPI_Info info);
 int MPI_File_get_view(MPI_File fh, MPI_Offset *disp, MPI_Datatype *etype, MPI_Datatype *filetype, char *datarep);
 int MPI_File_read_at_all(MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype, MPI_Status *status);
 int MPI_File_write_at(MPI_File fh, MPI_Offset offset, const void *buf, int count, MPI_Datatype datatype, MPI_Status *status);
@@ -108,8 +106,6 @@ int MPI_File_iread_at_all(MPI_File fh, MPI_Offset offset, void *buf, int count,
 int MPI_File_iwrite_at_all(MPI_File fh, MPI_Offset offset, const void *buf, int count, MPI_Datatype datatype, MPI_Request *request);
 int MPI_File_read(MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Status *status);
 int MPI_File_read_all(MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Status *status);
-int MPI_File_write(MPI_File fh, const void *buf, int count, MPI_Datatype datatype, MPI_Status *status);
-int MPI_File_write_all(MPI_File fh, const void *buf, int count, MPI_Datatype datatype, MPI_Status *status);
 int MPI_File_iread(MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Request *request);
 int MPI_File_iwrite(MPI_File fh, const void *buf, int count, MPI_Datatype datatype, MPI_Request *request);
 int MPI_File_iread_all(MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Request *request);
@@ -217,7 +213,6 @@ int MPI_Ireduce_scatter_block(const void *sendbuf, void *recvbuf, int recvcount,
 int MPI_Register_datarep(const char *datarep, MPI_Datarep_conversion_function *read_conversion_fn, MPI_Datarep_conversion_function *write_conversion_fn, MPI_Datarep_extent_function *dtype_file_extent_fn, void *extra_state);
 
 
-int MPI_Request_free(MPI_Request *request);
 int MPI_Rget(void *origin_addr, int origin_count, MPI_Datatype origin_datatype, int target_rank, MPI_Aint target_disp, int target_count, MPI_Datatype target_datatype, MPI_Win win, MPI_Request *request);
 int MPI_Rget_accumulate(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype, void *result_addr, int result_count, MPI_Datatype result_datatype, int target_rank, MPI_Aint target_disp, int target_count, MPI_Datatype target_datatype, MPI_Op op, MPI_Win win, MPI_Request *request);
 int MPI_Rput(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype, int target_rank, MPI_Aint target_disp, int target_cout, MPI_Datatype target_datatype, MPI_Win win, MPI_Request *request);
@@ -247,7 +242,6 @@ int MPI_Type_create_hindexed(int count, const int array_of_blocklengths[], const
 int MPI_Type_create_hvector(int count, int blocklength, MPI_Aint stride, MPI_Datatype oldtype, MPI_Datatype *newtype);
 int MPI_Type_create_keyval(MPI_Type_copy_attr_function *type_copy_attr_fn, MPI_Type_delete_attr_function *type_delete_attr_fn, int *type_keyval, void *extra_state);
 int MPI_Type_create_indexed_block(int count, int blocklength, const int array_of_displacements[], MPI_Datatype oldtype, MPI_Datatype *newtype);
-int MPI_Type_create_subarray(int ndims, const int size_array[], const int subsize_array[], const int start_array[], int order, MPI_Datatype oldtype, MPI_Datatype *newtype);
 int MPI_Type_create_resized(MPI_Datatype oldtype, MPI_Aint lb, MPI_Aint extent, MPI_Datatype *newtype);
 int MPI_Type_delete_attr(MPI_Datatype type, int type_keyval);
 int MPI_Type_dup(MPI_Datatype type, MPI_Datatype *newtype);
@@ -316,4 +310,3 @@ int MPI_Win_test(MPI_Win win, int *flag);
 int MPI_Win_unlock(int rank, MPI_Win win);
 int MPI_Win_unlock_all(MPI_Win win);
 int MPI_Win_wait(MPI_Win win);
-double MPI_Wtick(void);

mpi-proxy-split/mpi-wrappers/mpi_wrappers.cpp

@@ -77,6 +77,7 @@ USER_DEFINED_WRAPPER(int, Init_thread, (int *) argc, (char ***) argv,
 // later in the file.
 // DEFINE_FNC(int, Finalize, (void))
 DEFINE_FNC(double, Wtime, (void))
+DEFINE_FNC(double, Wtick, (void))
 DEFINE_FNC(int, Finalized, (int *) flag)
 DEFINE_FNC(int, Get_processor_name, (char *) name, (int *) resultlen)
 DEFINE_FNC(int, Initialized, (int *) flag)
@@ -115,6 +116,7 @@ PMPI_IMPL(int, MPI_Finalize, void)
 PMPI_IMPL(int, MPI_Finalized, int *flag)
 PMPI_IMPL(int, MPI_Get_processor_name, char *name, int *resultlen)
 PMPI_IMPL(double, MPI_Wtime, void)
+PMPI_IMPL(double, MPI_Wtick, void)
 PMPI_IMPL(int, MPI_Initialized, int *flag)
 PMPI_IMPL(int, MPI_Init_thread, int *argc, char ***argv,
           int required, int *provided)

mpi-proxy-split/record-replay.cpp

@@ -56,6 +56,7 @@ static int restoreTypeHVector(MpiRecord& rec);
 static int restoreTypeIndexed(MpiRecord& rec);
 static int restoreTypeFree(MpiRecord& rec);
 static int restoreTypeCreateStruct(MpiRecord& rec);
+static int restoreTypeCreateSubarray(MpiRecord& rec);
 
 static int restoreCartCreate(MpiRecord& rec);
 static int restoreCartMap(MpiRecord& rec);
@@ -205,6 +206,10 @@ dmtcp_mpi::restoreTypes(MpiRecord &rec)
       JTRACE("restoreTypeCreateStruct");
       rc = restoreTypeCreateStruct(rec);
       break;
+    case GENERATE_ENUM(Type_create_subarray):
+      JTRACE("restoreTypeCreateSubarray");
+      rc = restoreTypeCreateSubarray(rec);
+      break;
     default:
       JWARNING(false)(rec.getType()).Text("Unknown call");
       break;
@@ -615,6 +620,29 @@ restoreTypeCreateStruct(MpiRecord& rec)
   return retval;
 }
 
+static int
+restoreTypeCreateSubarray(MpiRecord& rec)
+{
+  int retval;
+  int ndims = rec.args(0);
+  const int *array_of_sizes = rec.args(1);
+  const int *array_of_subsizes = rec.args(2);
+  const int *array_of_starts = rec.args(3);
+  int order = rec.args(4);
+  MPI_Datatype oldtype = rec.args(5);
+  MPI_Datatype newtype = MPI_DATATYPE_NULL;
+  retval = FNC_CALL(Type_create_subarray, rec)(ndims, array_of_sizes,
+           array_of_subsizes, array_of_starts, order, oldtype, &newtype);
+  JWARNING(retval == MPI_SUCCESS)
+          .Text("Could not restore MPI subarray datatype");
+  if (retval == MPI_SUCCESS) {
+    MPI_Datatype virtType = rec.args(6);
+    UPDATE_TYPE_MAP(virtType, newtype);
+  }
+  return retval;
+}
+
+
 #ifdef SINGLE_CART_REORDER
 int
 load_cartesian_properties(const char *filename, CartesianProperties *cp)

mpi-proxy-split/test/Cancel_test.c

@@ -0,0 +1,48 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <mpi.h>
+#include <unistd.h>
+
+int main(int argc, char* argv[])
+{
+  MPI_Init(&argc, &argv);
+
+  int size, rank, buf;
+  MPI_Comm_size(MPI_COMM_WORLD, &size);
+  MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+
+  if(size != 2){
+    printf("The application works only with 2 processes.\n");
+    fflush(stdout);
+    MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE);
+  }
+
+  if(rank==0){
+    buf = 1;
+    MPI_Request req;
+    MPI_Status stat;
+
+    #if 0
+    MPI_Irecv(&buf, 1, MPI_INT, 1, 0, MPI_COMM_WORLD, &req);
+    MPI_Cancel(&req);
+    printf("Posted Irecv and cancelled request.\n");fflush(stdout);
+
+    #else
+    MPI_Isend(&buf, 1, MPI_INT, 1, 0, MPI_COMM_WORLD, &req);
+    MPI_Cancel(&req);
+    printf("Posted Isend and cancelled request.\n");fflush(stdout);
+
+    #endif
+
+    // Sleep
+    printf("Sleeping for 30 seconds.\n");fflush(stdout);
+    sleep(30);
+
+    MPI_Wait(&req, &stat);
+  } 
+  MPI_Barrier(MPI_COMM_WORLD);
+  if(rank==0){
+    printf("Finishing application.\n");fflush(stdout);
+  }
+  return 0;
+}