From 1e15d2aee5219087b6fc8a449bdc473e69bb4196 Mon Sep 17 00:00:00 2001
From: Hyeoksu Lee <hyeoksu@carpenter04.hsn.ex4000.erdc.hpc.mil>
Date: Thu, 20 Jun 2024 03:02:31 -0500
Subject: [PATCH 1/2] Support for DoD Carpenter

---
 mfc.sh                             | 10 +++---
 toolchain/bootstrap/modules.sh     |  7 +++--
 toolchain/modules                  |  5 ++-
 toolchain/templates/carpenter.mako | 49 ++++++++++++++++++++++++++++++
 toolchain/util.sh                  |  4 +--
 5 files changed, 64 insertions(+), 11 deletions(-)
 create mode 100644 toolchain/templates/carpenter.mako

diff --git a/mfc.sh b/mfc.sh
index 3bc5a2bd55..5797c0eecb 100755
--- a/mfc.sh
+++ b/mfc.sh
@@ -22,19 +22,19 @@ if [ -d "$(pwd)/bootstrap" ] || [ -d "$(pwd)/dependencies" ] || [ -f "$(pwd)/bui
 fi
 
 # If the user wishes to run the "load" script
-if [ "$1" == 'load' ]; then
+if [ "$1" '==' 'load' ]; then
     shift; . "$(pwd)/toolchain/bootstrap/modules.sh" $@; return
-elif [ "$1" == "lint" ]; then
+elif [ "$1" '==' "lint" ]; then
     . "$(pwd)/toolchain/bootstrap/python.sh"
 
     shift; . "$(pwd)/toolchain/bootstrap/lint.sh"    $@; exit 0
-elif [ "$1" == "format" ]; then
+elif [ "$1" '==' "format" ]; then
     . "$(pwd)/toolchain/bootstrap/python.sh"
 
     shift; . "$(pwd)/toolchain/bootstrap/format.sh"  $@; exit 0
-elif [ "$1" == "docker" ]; then
+elif [ "$1" '==' "docker" ]; then
     shift; . "$(pwd)/toolchain/bootstrap/docker.sh"  $@; exit 0
-elif [ "$1" == "venv" ]; then
+elif [ "$1" '==' "venv" ]; then
     shift; . "$(pwd)/toolchain/bootstrap/python.sh"  $@; return
 fi
 
diff --git a/toolchain/bootstrap/modules.sh b/toolchain/bootstrap/modules.sh
index 056df70b7d..b6cddad437 100644
--- a/toolchain/bootstrap/modules.sh
+++ b/toolchain/bootstrap/modules.sh
@@ -23,7 +23,8 @@ if [ -v $u_c ]; then
     log   "$C""ACCESS$W:  Bridges2   (b) | Expanse (e) | Delta  (d)"
     log   "$Y""Gatech$W:  Phoenix    (p)"
     log   "$R""Caltech$W: Richardson (r)"
-    log_n "($G""a$W/$G""f$W/$G""s$W/$G""w$W/$C""b$W/$C""e$CR/$C""d$CR/$Y""p$CR/$R""r$CR): "
+    log   "$B""DoD$W:     Carpenter  (c)"
+    log_n "($G""a$W/$G""f$W/$G""s$W/$G""w$W/$C""b$W/$C""e$CR/$C""d$CR/$Y""p$CR/$R""r$CR/$B""c$CR): "
     read u_c
     log
 fi
@@ -42,9 +43,9 @@ fi
 u_c=$(echo "$u_c" | tr '[:upper:]' '[:lower:]')
 u_cg=$(echo "$u_cg" | tr '[:upper:]' '[:lower:]')
 
-if [ "$u_cg" == 'c' ] || [ "$u_cg" == 'cpu' ]; then
+if [ "$u_cg" '==' 'c' ] || [ "$u_cg" '==' 'cpu' ]; then
     CG='CPU'; cg='cpu'
-elif [ "$u_cg" == "g" ] || [ "$u_cg" == 'gpu' ]; then
+elif [ "$u_cg" '==' "g" ] || [ "$u_cg" '==' 'gpu' ]; then
     CG='GPU'; cg='gpu'
 fi
 
diff --git a/toolchain/modules b/toolchain/modules
index e852818aa3..77e85ba300 100644
--- a/toolchain/modules
+++ b/toolchain/modules
@@ -54,9 +54,12 @@ f-all cray-fftw cray-hdf5 cray-mpich/8.1.26 cce/16.0.1
 f-all rocm/5.5.1 cray-python omniperf
 f-cpu
 
-
 d     NCSA Delta
 d-all python/3.11.6
 d-cpu gcc/11.4.0 openmpi
 d-gpu nvhpc/22.11 openmpi+cuda/4.1.5+cuda cmake
 d-gpu CC=nvc CXX=nvc++ FC=nvfortran
+
+c     DoD Carpenter
+c-all python
+c-cpu gcc/12.2.0 cmake/3.28.1-gcc-12.2.0 openmpi/4.1.6
diff --git a/toolchain/templates/carpenter.mako b/toolchain/templates/carpenter.mako
new file mode 100644
index 0000000000..6ffc6ae2d9
--- /dev/null
+++ b/toolchain/templates/carpenter.mako
@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+
+<%namespace name="helpers" file="helpers.mako"/>
+
+% if engine == 'batch':
+#PBS -l select=${nodes}:ncpus=192:mpiprocs=${tasks_per_node}
+#PBS -N "${name}"
+#PBS -l walltime=${walltime}
+% if partition:
+#PBS -q ${partition}
+% endif
+% if account:
+#PBS -A ${account}
+% endif
+% if email:
+#PBS -M ${email}
+#PBS -m abe
+% endif
+#PBS -o "${name}.out"
+#PBS -e "${name}.err"
+#PBS -V
+% endif
+
+${helpers.template_prologue()}
+
+ok ":) Loading modules:\n"
+cd "${MFC_ROOTDIR}"
+. ./mfc.sh load -c c -m ${'g' if gpu else 'c'}
+cd - > /dev/null
+echo
+
+    
+% for target in targets:
+    ${helpers.run_prologue(target)}
+
+    % if not mpi:
+        (set -x; ${profiler} "${target.get_install_binpath(case)}")
+    % else:
+        (set -x; ${profiler}                              \
+            mpirun -np ${nodes*tasks_per_node}            \
+                   "${target.get_install_binpath(case)}")
+    % endif
+
+    ${helpers.run_epilogue(target)}
+
+    echo
+% endfor
+
+${helpers.template_epilogue()}
diff --git a/toolchain/util.sh b/toolchain/util.sh
index 5e32eee2f9..c67c9cad66 100644
--- a/toolchain/util.sh
+++ b/toolchain/util.sh
@@ -2,10 +2,10 @@
 
 if [ -t 1 ]; then
     RED="\x1B[31m";    CYAN="\x1B[36m";    GREEN="\x1B[32m"
-    YELLOW="\x1B[33m"; MAGENTA="\x1B[35m"; COLOR_RESET="\033[m"
+    YELLOW="\x1B[33m"; MAGENTA="\x1B[35m"; BLUE="\x1B[34m"; COLOR_RESET="\033[m"
 
     R=$RED;    C=$CYAN;    G=$GREEN
-    Y=$YELLOW; M=$MAGENTA; CR=$COLOR_RESET; W=$CR
+    Y=$YELLOW; M=$MAGENTA; B=$BLUE; CR=$COLOR_RESET; W=$CR
 fi
 
 log()   { echo -e  "$CYAN"mfc"$COLOR_RESET: $1$COLOR_RESET"; }

From e35017f2d3c742182fadc793cfe324d60caab992 Mon Sep 17 00:00:00 2001
From: Hyeoksu Lee <hyeoksu@carpenter03.hsn.ex4000.erdc.hpc.mil>
Date: Thu, 20 Jun 2024 18:05:41 -0500
Subject: [PATCH 2/2] Add GPU modules

---
 toolchain/modules | 1 +
 1 file changed, 1 insertion(+)

diff --git a/toolchain/modules b/toolchain/modules
index 77e85ba300..f517fc8f19 100644
--- a/toolchain/modules
+++ b/toolchain/modules
@@ -63,3 +63,4 @@ d-gpu CC=nvc CXX=nvc++ FC=nvfortran
 c     DoD Carpenter
 c-all python
 c-cpu gcc/12.2.0 cmake/3.28.1-gcc-12.2.0 openmpi/4.1.6
+c-gpu nvhpc/23.7 cuda/12.2