From e31ffc513f478d56e03bf148b02f3f2d8468ac77 Mon Sep 17 00:00:00 2001
From: Aurelien Bouteiller <bouteill@icl.utk.edu>
Date: Mon, 22 Jan 2024 15:55:51 -0500
Subject: [PATCH] bugfix: cuda handles were not consistently initialized when
 gpus where initialized/not initialized differently than the -g param

Signed-off-by: Aurelien Bouteiller <bouteill@icl.utk.edu>
---
 src/dplasmaaux_cuda.c      |  9 +++++++++
 src/dplasmaaux_cuda.h      |  1 +
 src/dtd_wrappers/zgemm.c   |  1 +
 tests/common.c             | 20 +++++---------------
 tests/testing_zgemm_dtd.c  | 15 ---------------
 tests/testing_zpotrf_dtd.c |  4 ----
 6 files changed, 16 insertions(+), 34 deletions(-)

diff --git a/src/dplasmaaux_cuda.c b/src/dplasmaaux_cuda.c
index e90cf82a..e4c0ceb8 100644
--- a/src/dplasmaaux_cuda.c
+++ b/src/dplasmaaux_cuda.c
@@ -97,3 +97,12 @@ void *dplasma_create_cuda_handles(void *obj, void *_n)
 
     return new;
 }
+
+void dplasma_destroy_cuda_handles(void *_h, void *_n)
+{
+    dplasma_cuda_handles_t *handles = (dplasma_cuda_handles_t*)_h;
+    (void)_n;
+    cublasDestroy_v2(handles->cublas_handle);
+    cusolverDnDestroy(handles->cusolverDn_handle);
+    free(handles);
+}
diff --git a/src/dplasmaaux_cuda.h b/src/dplasmaaux_cuda.h
index f7e57ed2..aa1360ff 100644
--- a/src/dplasmaaux_cuda.h
+++ b/src/dplasmaaux_cuda.h
@@ -66,6 +66,7 @@ typedef struct {
 } dplasma_cuda_handles_t;
 
 void *dplasma_create_cuda_handles(void *obj, void *user);
+void dplasma_destroy_cuda_handles(void *_h, void *_n);
 
 #define DPLASMA_CUBLAS_CHECK_STATUS( STR, STATUS, CODE )                     \
     do {                                                                     \
diff --git a/src/dtd_wrappers/zgemm.c b/src/dtd_wrappers/zgemm.c
index 80361892..9b39354d 100644
--- a/src/dtd_wrappers/zgemm.c
+++ b/src/dtd_wrappers/zgemm.c
@@ -81,6 +81,7 @@ parsec_core_zgemm_cuda(parsec_device_gpu_module_t* gpu_device,
 #endif /* defined(PARSEC_DEBUG_NOISIER) */
 
     handles = parsec_info_get(&gpu_stream->infos, CuHI);
+    assert(NULL != handles);
 
     parsec_cuda_exec_stream_t* cuda_stream = (parsec_cuda_exec_stream_t*)gpu_stream;
     cublasSetStream( handles->cublas_handle, cuda_stream->cuda_stream );
diff --git a/tests/common.c b/tests/common.c
index 78a3d865..3b7c0fa9 100644
--- a/tests/common.c
+++ b/tests/common.c
@@ -626,17 +626,6 @@ char cwd[1024];
 int unix_timestamp;
 #endif
 
-#if defined(DPLASMA_HAVE_CUDA)
-static void destroy_cuda_handles(void *_h, void *_n)
-{
-    dplasma_cuda_handles_t *handles = (dplasma_cuda_handles_t*)_h;
-    (void)_n;
-    cublasDestroy_v2(handles->cublas_handle);
-    cusolverDnDestroy(handles->cusolverDn_handle);
-    free(handles);
-}
-#endif
-
 parsec_context_t* setup_parsec(int argc, char **argv, int *iparam)
 {
 #ifdef PARSEC_PROF_TRACE
@@ -716,10 +705,11 @@ parsec_context_t* setup_parsec(int argc, char **argv, int *iparam)
         }
     }
     if( nbgpu > 0 ) {
-        parsec_info_register(&parsec_per_stream_infos, "DPLASMA::CUDA::HANDLES",
-                             destroy_cuda_handles, NULL,
-                             dplasma_create_cuda_handles, NULL,
-                             NULL);
+        CuHI = parsec_info_register(&parsec_per_stream_infos, "DPLASMA::CUDA::HANDLES",
+                                   dplasma_destroy_cuda_handles, NULL,
+                                   dplasma_create_cuda_handles, NULL,
+                                   NULL);
+        assert(-1 != CuHI);
     }
 #endif
 
diff --git a/tests/testing_zgemm_dtd.c b/tests/testing_zgemm_dtd.c
index fb11a847..3a02a216 100644
--- a/tests/testing_zgemm_dtd.c
+++ b/tests/testing_zgemm_dtd.c
@@ -123,13 +123,6 @@ int main(int argc, char ** argv)
         parsec_context_start(parsec);
         parsec_task_class_t *zgemm_tc = parsec_dtd_create_zgemm_task_class(dtd_tp, TILE_FULL, PARSEC_DEV_ALL);
 
-#if defined(DPLASMA_HAVE_CUDA)
-        if( gpus > 0 ){
-            CuHI = parsec_info_lookup(&parsec_per_stream_infos, "DPLASMA::CUDA::HANDLES", NULL);
-            assert( CuHI != -1);
-        }
-#endif /* defined(DPLASMA_HAVE_CUDA) */
-
         for( m = 0; m < dcC.super.mt; m++ ) {
             tempmm = m == dcC.super.mt-1 ? dcC.super.m-m*dcC.super.mb : dcC.super.mb;
             ldcm = BLKLDD(&dcC.super, m);
@@ -370,14 +363,6 @@ int main(int argc, char ** argv)
                 parsec_context_start(parsec);
                 parsec_task_class_t *zgemm_tc = parsec_dtd_create_zgemm_task_class(dtd_tp, TILE_FULL, PARSEC_DEV_ALL);
 
-#if defined(DPLASMA_HAVE_CUDA)
-                if( gpus > 0 ){
-                    CuHI = parsec_info_lookup(&parsec_per_stream_infos, "DPLASMA::CUDA::HANDLES", NULL);
-                    assert( CuHI != -1);
-                }
-#endif
-
-
                 for( m = 0; m < dcC.super.mt; m++ ) {
                     tempmm = m == dcC.super.mt-1 ? dcC.super.m-m*dcC.super.mb : dcC.super.mb;
                     ldcm = BLKLDD(&dcC.super, m);
diff --git a/tests/testing_zpotrf_dtd.c b/tests/testing_zpotrf_dtd.c
index 423f1cf8..a94325ef 100644
--- a/tests/testing_zpotrf_dtd.c
+++ b/tests/testing_zpotrf_dtd.c
@@ -96,10 +96,6 @@ int main(int argc, char **argv)
 
 #if defined(DPLASMA_HAVE_CUDA)
     zpotrf_dtd_workspace_info_t* infos = (zpotrf_dtd_workspace_info_t*) malloc(sizeof(zpotrf_dtd_workspace_info_t));
-    if( gpus > 0 ){
-        CuHI = parsec_info_lookup(&parsec_per_stream_infos, "DPLASMA::CUDA::HANDLES", NULL);
-        assert(CuHI != -1);
-    }
 #endif
 
     if( dplasmaLower == uplo ) {