In multi-GPU runs, we need to set the target GPU before calling the u…

…ser's object info constructor (and potentially destructor). Allow the device to define a function to do so in order to keep the user code unchanged. This adds an additional function pointer call, but this call only happens once per info and per stream, ideally during warmup/initialization time

parsec/class/info.c

@@ -125,6 +125,7 @@ parsec_info_id_t parsec_info_unregister(parsec_info_t *nfo, parsec_info_id_t iid
                     item2 = PARSEC_LIST_ITERATOR_NEXT(item2)) {
                     ioa = (parsec_info_object_array_t*)item2;
                     if(iid < ioa->known_infos && NULL != ioa->info_objects[iid]) {
+                        if(NULL != ioa->ctx_set) ioa->ctx_set(ioa->ctx_set_obj);
                         ie->destructor(ioa->info_objects[iid], ie->des_data);
                         ioa->info_objects[iid] = NULL;
                     }
@@ -210,7 +211,7 @@ static void parsec_info_object_array_constructor(parsec_object_t *obj)
 /* The constructor cannot set the info, as it does not take additional
  * parameters. Thus, it is needed to call init after constructing the
  * info_object_array. */
-void parsec_info_object_array_init(parsec_info_object_array_t *oa, parsec_info_t *nfo, void *cons_obj)
+void parsec_info_object_array_init(parsec_info_object_array_t *oa, parsec_info_t *nfo, void *cons_obj, parsec_info_set_ctx_fn ctx_set, void *ctx_set_param)
 {
     oa->known_infos = nfo->max_id+1;
     parsec_list_push_front(&nfo->ioa_list, &oa->list_item);
@@ -220,6 +221,8 @@ void parsec_info_object_array_init(parsec_info_object_array_t *oa, parsec_info_t
         oa->info_objects = calloc(sizeof(void*), oa->known_infos);
     oa->infos = nfo;
     oa->cons_obj = cons_obj;
+    oa->ctx_set = ctx_set;
+    oa->ctx_set_obj = ctx_set_param;
 }
 
 static void parsec_info_object_array_destructor(parsec_object_t *obj)
@@ -311,6 +314,7 @@ void *parsec_info_get(parsec_info_object_array_t *oa, parsec_info_id_t iid)
     ie = parsec_info_lookup_by_iid(oa->infos, iid);
     if(NULL == ie->constructor)
         return ret;
+    if(NULL != oa->ctx_set) oa->ctx_set(oa->ctx_set_obj);
     nio = ie->constructor(oa->cons_obj, ie->cons_data);
     ret = parsec_info_test_and_set(oa, iid, nio, NULL);
     if(ret != nio && NULL != ie->destructor) {

parsec/class/info.h

@@ -82,6 +82,20 @@ typedef void (*parsec_info_destructor_t)(void *elt, void *cb_data);
  */
 typedef void *(*parsec_info_constructor_t)(void *obj, void *cb_data);
 
+/**
+ * @brief Prototype of a context setter function
+ *
+ * @details
+ *   A function with this prototype will be called each time a
+ *   user callback is called, before it is called, to set the
+ *   context in which the user callback is supposed to execute
+ *   (this is dependent on the level of software that defines
+ *   the info)
+ *
+ *   @param[inout] cb_data: opaque pointer provided by the user
+ */
+typedef int (*parsec_info_set_ctx_fn)(void *cb_data);
+
 /**
  * @brief The descriptor of a single info
  */
@@ -100,7 +114,7 @@ struct parsec_info_entry_s {
 
 /**
  * @brief An array of info objects
- * 
+ *
  * @details This structure holds the info objects
  *   in an array indexed by the iid
  */
@@ -113,6 +127,8 @@ struct parsec_info_object_array_s {
     void                 **info_objects; /**< Info objects are stored in this array indexed by
                                           *   info_entry->iid */
     void                  *cons_obj;     /**< obj pointer for the constructor */
+    parsec_info_set_ctx_fn ctx_set;      /**< Callback to set the context of the ioa */
+    void                  *ctx_set_obj;  /**< Parameter passed to ctx_set */
 };
 
 PARSEC_OBJ_CLASS_DECLARATION(parsec_info_object_array_t);
@@ -183,14 +199,17 @@ parsec_info_id_t parsec_info_lookup(parsec_info_t *nfo, const char *name, void *
  *   @param[IN]      nfo: the info collection that defines its keys
  *   @param[IN] cons_obj: pointer to the object holding the object array (passed as
  *                        first argument to constructor calls)
+ *   @param[IN] ctx_set:  how to set the context for this ioa
+ *   @param[IN] ctx_set_param: what parameter to pass to ctx_set
  *
  * @remark when constructing an object array with PARSE_OBJ_CONSTRUCT or PARSEC_OBJ_NEW,
  *   the parsec_info_t cannot be associated to the object array automatically as the constructor
  *   do not take parameters. It is thus needed to initialize the object_array with the info
  *   after it is constructed.
  */
 void parsec_info_object_array_init(parsec_info_object_array_t *oa, parsec_info_t *nfo,
-                                   void *cons_obj);
+                                   void *cons_obj, parsec_info_set_ctx_fn ctx_set,
+                                   void *ctx_set_obj);
 
 /**
  * @brief Set an info in an array of objects

parsec/mca/device/cuda/device_cuda_module.c

@@ -470,7 +470,7 @@ parsec_cuda_module_init( int dev_id, parsec_device_module_t** module )
                                  {goto release_device;} );
         exec_stream->workspace    = NULL;
         PARSEC_OBJ_CONSTRUCT(&exec_stream->infos, parsec_info_object_array_t);
-        parsec_info_object_array_init(&exec_stream->infos, &parsec_per_stream_infos, exec_stream);
+        parsec_info_object_array_init(&exec_stream->infos, &parsec_per_stream_infos, exec_stream, (parsec_info_set_ctx_fn)parsec_cuda_set_device, gpu_device); 
         exec_stream->max_events   = PARSEC_MAX_EVENTS_PER_STREAM;
         exec_stream->executed     = 0;
         exec_stream->start        = 0;

parsec/mca/device/device.c

@@ -14,6 +14,7 @@
 #include "parsec/execution_stream.h"
 #include "parsec/utils/argv.h"
 #include "parsec/parsec_internal.h"
+#include "parsec/mca/device/device_gpu.h"
 
 #include <stdlib.h>
 #if defined(PARSEC_HAVE_ERRNO_H)
@@ -1022,6 +1023,17 @@ int parsec_mca_device_attach(parsec_context_t* context)
     return PARSEC_SUCCESS;
 }
 
+static int parsec_device_set_ctx(void *_device)
+{
+    parsec_device_module_t *device = (parsec_device_module_t*)_device;
+    if(PARSEC_DEV_IS_GPU(device->type)) {
+        parsec_device_gpu_module_t *gpu_device = (parsec_device_gpu_module_t *)device;
+        gpu_device->set_device(gpu_device);
+    }
+    /* Nothing to do for non-GPU devices */
+    return PARSEC_SUCCESS;
+}
+
 int parsec_mca_device_add(parsec_context_t* context, parsec_device_module_t* device)
 {
     if( parsec_mca_device_are_freezed ) {
@@ -1045,7 +1057,7 @@ int parsec_mca_device_add(parsec_context_t* context, parsec_device_module_t* dev
     device->context = context;
     parsec_atomic_unlock(&parsec_devices_mutex);  /* CRITICAL SECTION: END */
     PARSEC_OBJ_CONSTRUCT(&device->infos, parsec_info_object_array_t);
-    parsec_info_object_array_init(&device->infos, &parsec_per_device_infos, device);
+    parsec_info_object_array_init(&device->infos, &parsec_per_device_infos, device, parsec_device_set_ctx, device);
     return device->device_index;
 }
 

parsec/mca/device/level_zero/device_level_zero_module.c

@@ -308,7 +308,7 @@ int parsec_level_zero_module_init( int dev_id, parsec_device_level_zero_driver_t
         PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandQueueCreate ", ze_rc, {goto release_device;} );
         exec_stream->workspace    = NULL;
         PARSEC_OBJ_CONSTRUCT(&exec_stream->infos, parsec_info_object_array_t);
-        parsec_info_object_array_init(&exec_stream->infos, &parsec_per_stream_infos, exec_stream);
+        parsec_info_object_array_init(&exec_stream->infos, &parsec_per_stream_infos, exec_stream, (parsec_info_set_ctx_fn)parsec_level_zero_set_device, gpu_device);
         exec_stream->max_events   = PARSEC_MAX_EVENTS_PER_STREAM;
         exec_stream->executed     = 0;
         exec_stream->start        = 0;