From 39e410a4924664ce56d47d58cf9c320eaab20f5f Mon Sep 17 00:00:00 2001
From: Max Hawkins <37495064+max-Hawkins@users.noreply.github.com>
Date: Thu, 7 Nov 2024 14:17:50 -0500
Subject: [PATCH] Specify CUDA version for login node builds (#697)

Co-authored-by: Spencer Bryngelson <shb@gatech.edu>
Co-authored-by: Spencer Bryngelson <sbryngelson@gmail.com>
---
 CMakeLists.txt                 | 14 +++++++-------
 toolchain/bootstrap/modules.sh |  4 ++--
 toolchain/modules              |  2 ++
 3 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index c0bb7616b..22268e250 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -127,7 +127,7 @@ if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
 
     if (CMAKE_BUILD_TYPE STREQUAL "Debug")
         add_compile_options(
-            -Wall 
+            -Wall
             -fcheck=all,no-array-temps
             -fbacktrace
             -fimplicit-none
@@ -142,7 +142,7 @@ if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
         add_compile_options(
             $<$<COMPILE_LANGUAGE:Fortran>:-fallow-invalid-boz>
             $<$<COMPILE_LANGUAGE:Fortran>:-fallow-argument-mismatch>
-            $<$<COMPILE_LANGUAGE:Fortran>:-fcheck=bounds>            
+            $<$<COMPILE_LANGUAGE:Fortran>:-fcheck=bounds>
         )
     endif()
 elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "Cray")
@@ -155,7 +155,7 @@ elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "Cray")
     )
 
     add_link_options("SHELL:-hkeepfiles")
-    
+
     if (CMAKE_BUILD_TYPE STREQUAL "Debug")
         add_compile_options(
                 "SHELL:-h acc_model=auto_async_none"
@@ -197,7 +197,7 @@ elseif ((CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC") OR (CMAKE_Fortran_COMPILER_
 
     if (DEFINED ENV{MFC_CUDA_CC})
         string(REGEX MATCHALL "[0-9]+" MFC_CUDA_CC $ENV{MFC_CUDA_CC})
-        message(STATUS "Found $MFC_CUDA_CC specified. GPU code will be generated for ${MFC_CUDA_CC}.")
+        message(STATUS "Found $MFC_CUDA_CC specified. GPU code will be generated for compute capability(ies) ${MFC_CUDA_CC}.")
     endif()
 endif()
 
@@ -231,7 +231,7 @@ if (CMAKE_BUILD_TYPE STREQUAL "Release")
 	    else()
             message(STATUS "IPO / LTO is NOT available")
         endif()
-    endif() 
+    endif()
 endif()
 
 if (CMAKE_BUILD_TYPE STREQUAL "Debug")
@@ -295,7 +295,7 @@ macro(HANDLE_SOURCES target useCommon)
 
     string(TOUPPER ${target} ${target}_UPPER)
 
-    # Gather: 
+    # Gather:
     # *          src/[<target>,(common)]/*.f90
     # * (if any) <build>/modules/<target>/*.f90
     file(GLOB ${target}_F90s CONFIGURE_DEPENDS "${${target}_DIR}/*.f90"
@@ -388,7 +388,7 @@ function(MFC_SETUP_TARGET)
     # A little hacky, but it *is* an edge-case for *one* compiler.
     if (NVHPC_USE_TWO_PASS_IPO)
         add_library(${ARGS_TARGET}_lib OBJECT ${ARGS_SOURCES})
-        target_compile_options(${ARGS_TARGET}_lib PRIVATE 
+        target_compile_options(${ARGS_TARGET}_lib PRIVATE
 		    $<$<COMPILE_LANGUAGE:Fortran>:-Mextract=lib:${ARGS_TARGET}_lib>
 	        $<$<COMPILE_LANGUAGE:Fortran>:-Minline>
     	)
diff --git a/toolchain/bootstrap/modules.sh b/toolchain/bootstrap/modules.sh
index 3c83d9fdc..db964c436 100644
--- a/toolchain/bootstrap/modules.sh
+++ b/toolchain/bootstrap/modules.sh
@@ -91,8 +91,8 @@ if ! module load $MODULES; then
 fi
 
 if [ $(echo "$VARIABLES" | grep = | wc -c) -gt 0 ]; then
-    log " $ export $VARIABLES"
-    export $VARIABLES > /dev/null
+    log " $ export $(eval "echo $VARIABLES")"
+    export $(eval "echo $VARIABLES") > /dev/null
 fi
 
 # Don't check for Cray paths on Carpenter, otherwise do check if they exist
diff --git a/toolchain/modules b/toolchain/modules
index a2c558df4..2d0ebe925 100644
--- a/toolchain/modules
+++ b/toolchain/modules
@@ -45,6 +45,7 @@ p     GT Phoenix
 p-all python/3.10.10
 p-cpu gcc/12.3.0 openmpi/4.1.5
 p-gpu nvhpc/24.5 hpcx/2.19-cuda cuda/12.1.1
+p-gpu MFC_CUDA_CC=70,80,89,90 NVHPC_CUDA_HOME=$CUDA_HOME CC=nvc CXX=nvc++ FC=nvfortran
 
 f     OLCF Frontier
 f-all cce/18.0.0 cpe/24.07 rocm/6.1.3 cray-mpich/8.1.28
@@ -56,6 +57,7 @@ d-all python/3.11.6
 d-cpu gcc/11.4.0 openmpi
 d-gpu nvhpc/24.1 cuda/12.3.0 openmpi/4.1.5+cuda cmake
 d-gpu CC=nvc CXX=nvc++ FC=nvfortran
+d-gpu MFC_CUDA_CC=80,86
 
 c     DoD Carpenter
 c-all python/3.12.1