From b2409d0eadf6c06a71a1a98d44c3bbbc81de9ba9 Mon Sep 17 00:00:00 2001 From: Henry LE BERRE Date: Sat, 13 Jan 2024 19:55:10 -0500 Subject: [PATCH] Address Comments --- docs/documentation/running.md | 4 +-- misc/run-phoenix-release-cpu.sh | 2 +- misc/run-phoenix-release-gpu.sh | 2 +- toolchain/mfc/args.py | 1 + toolchain/templates/bridges2.mako | 14 ++++---- toolchain/templates/default.mako | 54 +++++++++++++++++++------------ toolchain/templates/phoenix.mako | 19 +++++++---- toolchain/templates/summit.mako | 28 +++++++++------- 8 files changed, 74 insertions(+), 50 deletions(-) diff --git a/docs/documentation/running.md b/docs/documentation/running.md index 566866f7d7..ceaa4a0f1b 100644 --- a/docs/documentation/running.md +++ b/docs/documentation/running.md @@ -17,8 +17,8 @@ several supercomputer clusters, both interactively and through batch submission. > > Adding a new template file or modifying an existing one will most likely be required if: > - You are on a cluster that does not have a template yet. -> - Your cluster is configured with SLURM and but fails when interactive jobs are -> launched with `mpirun`. +> - Your cluster is configured with SLURM but interactive job launches fail when +> using `srun`. You might need to invoke `mpirun` instead. > - Something in the existing default or computer template file is incompatible with > your system or does not provide a feature you need. > diff --git a/misc/run-phoenix-release-cpu.sh b/misc/run-phoenix-release-cpu.sh index 185d95865d..aea1b80a46 100644 --- a/misc/run-phoenix-release-cpu.sh +++ b/misc/run-phoenix-release-cpu.sh @@ -12,4 +12,4 @@ cd "$SLURM_SUBMIT_DIR" echo "Running in $(pwd):" . ./mfc.sh load -c p -m gpu -./mfc.sh test -j $(nproc) -a -- -b mpirun +./mfc.sh test -j $(nproc) -a -- -c phoenix diff --git a/misc/run-phoenix-release-gpu.sh b/misc/run-phoenix-release-gpu.sh index 45f16eb817..b2f3e4c1f8 100644 --- a/misc/run-phoenix-release-gpu.sh +++ b/misc/run-phoenix-release-gpu.sh @@ -19,4 +19,4 @@ set -x gpu_count=$(nvidia-smi -L | wc -l) # number of GPUs on node gpu_ids=$(seq -s ' ' 0 $(($gpu_count-1))) # 0,1,2,...,gpu_count-1 -./mfc.sh test -a -j 2 --gpu -g $gpu_ids -- -b mpirun +./mfc.sh test -a -j 2 --gpu -g $gpu_ids -- -c phoenix diff --git a/toolchain/mfc/args.py b/toolchain/mfc/args.py index 1855d01057..4e404be255 100644 --- a/toolchain/mfc/args.py +++ b/toolchain/mfc/args.py @@ -111,6 +111,7 @@ def add_common_arguments(p, mask = None): run.add_argument("-@", "--email", metavar="EMAIL", type=str, default="", help="(Batch) Email for job notification.") run.add_argument("-#", "--name", metavar="NAME", type=str, default="MFC", help="(Batch) Job name.") run.add_argument("-s", "--scratch", action="store_true", default=False, help="Build from scratch.") + run.add_argument("-b", "--binary", choices=["mpirun", "jsrun", "srun", "mpiexec"], type=str, default=None, help="(Interactive) Override MPI execution binary") run.add_argument("--ncu", nargs=argparse.REMAINDER, type=str, help="Profile with NVIDIA Nsight Compute.") run.add_argument("--nsys", nargs=argparse.REMAINDER, type=str, help="Profile with NVIDIA Nsight Systems.") run.add_argument( "--dry-run", action="store_true", default=False, help="(Batch) Run without submitting batch file.") diff --git a/toolchain/templates/bridges2.mako b/toolchain/templates/bridges2.mako index 203a8c3e6f..ed364bcf2b 100644 --- a/toolchain/templates/bridges2.mako +++ b/toolchain/templates/bridges2.mako @@ -27,27 +27,29 @@ <%include file="prologue.mako"/> -echo -e ":) Loading modules:\n" +ok ":) Loading modules:\n" cd "${rootdir}" . ./mfc.sh load -c b -m ${'g' if gpu else 'c'} cd - > /dev/null echo % for binpath in binpaths: - echo -e ":) Running ${binpath.split('/')[-1]}:\n" + ok -e ":) Running ${binpath.split('/')[-1]}:\n" % if not mpi: ${' '.join([f"'{x}'" for x in profiler ])} "${binpath}" % else: - mpirun -np ${nodes*tasks_per_node} \ - ${' '.join([f"'{x}'" for x in profiler ])} \ - "${binpath}" + ${' '.join([f"'{x}'" for x in profiler ])} \ + mpirun -np ${nodes*tasks_per_node} \ + "${binpath}" % endif % if engine == 'interactive': code=$? if [ $code -ne 0 ]; then - echo -e "\n:( $MAGENTA${binpath}$COLOR_RESET failed with exit code $MAGENTA$code$COLOR_RESET.\n" + echo + error ":( $MAGENTA${binpath}$COLOR_RESET failed with exit code $MAGENTA$code$COLOR_RESET." + echo exit 1 fi % endif diff --git a/toolchain/templates/default.mako b/toolchain/templates/default.mako index 32cf69f3ef..cb2414f22c 100644 --- a/toolchain/templates/default.mako +++ b/toolchain/templates/default.mako @@ -12,38 +12,50 @@ warn "This is the$MAGENTA default$COLOR_RESET template." warn "It is not intended to support all systems and execution engines." warn "Please use a different template via the $MAGENTA--computer$COLOR_RESET option." -echo + +# Find a suitable MPI launcher and store it in the variable "binary". +for binary in ${binary or ''} jsrun srun mpirun mpiexec; do + if command -v $binary > /dev/null; then + break + fi +done + +if ! command -v ${binary} > /dev/null; then + error ":( Could not find a suitable MPI launcher.\n" + exit 1 +else + ok ":) Selected MPI launcher $MAGENTA$binary$COLOR_RESET. Use$MAGENTA --binary$COLOR_RESET to override." +fi % for binpath in binpaths: - echo -e ":) Running $MAGENTA${binpath}$COLOR_RESET:\n" + ok ":) Running $MAGENTA${binpath}$COLOR_RESET:\n" % if not mpi: ${' '.join([f"'{x}'" for x in profiler ])} "${binpath}" % else: - if command -v jsrun > /dev/null; then - jsrun --nrs ${tasks_per_node*nodes} \ - --cpu_per_rs 1 \ - --gpu_per_rs ${1 if gpu else 0} \ - --tasks_per_rs 1 \ - ${' '.join([f"'{x}'" for x in profiler ])} \ - "${binpath}" - elif command -v srun > /dev/null; then - srun --ntasks-per-node ${tasks_per_node} \ - ${' '.join([f"'{x}'" for x in profiler ])} \ - "${binpath}" - elif command -v mpirun > /dev/null; then - mpirun -np ${nodes*tasks_per_node} \ - ${' '.join([f"'{x}'" for x in profiler ])} \ - "${binpath}" - else - echo -e "\n:( Could not find a suitable MPI launcher.\n" - exit 1 + if [ "$binary" == "jsrun" ]; then + ${' '.join([f"'{x}'" for x in profiler ])} \ + jsrun --nrs ${tasks_per_node*nodes} \ + --cpu_per_rs 1 \ + --gpu_per_rs ${1 if gpu else 0} \ + --tasks_per_rs 1 \ + "${binpath}" + elif [ "$binary" == "srun" ]; then + ${' '.join([f"'{x}'" for x in profiler ])} \ + srun --ntasks-per-node ${tasks_per_node} \ + "${binpath}" + elif [ "$binary" == "mpirun" ] || [ "$binary" == "mpiexec" ]; then + ${' '.join([f"'{x}'" for x in profiler ])} \ + $binary -np ${nodes*tasks_per_node} \ + "${binpath}" fi % endif code=$? if [ $code -ne 0 ]; then - echo -e "\n:( $MAGENTA${binpath}$COLOR_RESET failed with exit code $MAGENTA$code$COLOR_RESET.\n" + echo + error ":( $MAGENTA${binpath}$COLOR_RESET failed with exit code $MAGENTA$code$COLOR_RESET." + echo exit 1 fi diff --git a/toolchain/templates/phoenix.mako b/toolchain/templates/phoenix.mako index 495c96a1ef..72b2e339db 100644 --- a/toolchain/templates/phoenix.mako +++ b/toolchain/templates/phoenix.mako @@ -27,25 +27,30 @@ <%include file="prologue.mako"/> -echo -e ":) Loading modules:\n" -cd "${rootdir}" && . ./mfc.sh load -c p -m ${'g' if gpu else 'c'} && cd - +ok ":) Loading modules:\n" +cd "${rootdir}" +. ./mfc.sh load -c p -m ${'g' if gpu else 'c'} +cd - > /dev/null echo % for binpath in binpaths: - echo -e ":) Running ${binpath.split('/')[-1]}:\n" + ok ":) Running ${binpath.split('/')[-1]}:\n" % if not mpi: ${' '.join([f"'{x}'" for x in profiler ])} "${binpath}" % else: - mpirun -np ${nodes*tasks_per_node} \ - ${' '.join([f"'{x}'" for x in profiler ])} \ - "${binpath}" + ${' '.join([f"'{x}'" for x in profiler ])} \ + mpirun -np ${nodes*tasks_per_node} \ + --bind-to none \ + "${binpath}" % endif % if engine == 'interactive': code=$? if [ $code -ne 0 ]; then - echo -e "\n:( $MAGENTA${binpath}$COLOR_RESET failed with exit code $MAGENTA$code$COLOR_RESET.\n" + echo + error ":( $MAGENTA${binpath}$COLOR_RESET failed with exit code $MAGENTA$code$COLOR_RESET." + echo exit 1 fi % endif diff --git a/toolchain/templates/summit.mako b/toolchain/templates/summit.mako index cce6b70627..08a80eb61a 100644 --- a/toolchain/templates/summit.mako +++ b/toolchain/templates/summit.mako @@ -12,30 +12,34 @@ <%include file="prologue.mako"/> -echo -e ":) Loading modules:\n" -cd "${rootdir}" && . ./mfc.sh load -c s -m ${'g' if gpu else 'c'} && cd - +ok ":) Loading modules:\n" +cd "${rootdir}" +. ./mfc.sh load -c s -m ${'g' if gpu else 'c'} +cd - > /dev/null echo % for binpath in binpaths: - echo -e ":) Running ${binpath}:\n" + ok ":) Running ${binpath}:\n" % if not mpi: ${' '.join([f"'{x}'" for x in profiler ])} "${binpath}" % else: - jsrun \ - ${'--smpiargs="-gpu"' if gpu else ''} \ - --nrs ${tasks_per_node*nodes} \ - --cpu_per_rs 1 \ - --gpu_per_rs ${1 if gpu else 0} \ - --tasks_per_rs 1 \ - ${' '.join([f"'{x}'" for x in profiler ])} \ - "${binpath}" + ${' '.join([f"'{x}'" for x in profiler ])} \ + jsrun \ + ${'--smpiargs="-gpu"' if gpu else ''} \ + --nrs ${tasks_per_node*nodes} \ + --cpu_per_rs 1 \ + --gpu_per_rs ${1 if gpu else 0} \ + --tasks_per_rs 1 \ + "${binpath}" % endif % if engine == 'interactive': code=$? if [ $code -ne 0 ]; then - echo -e "\n:( $MAGENTA${binpath}$COLOR_RESET failed with exit code $MAGENTA$code$COLOR_RESET.\n" + echo + error ":( $MAGENTA${binpath}$COLOR_RESET failed with exit code $MAGENTA$code$COLOR_RESET." + echo exit 1 fi % endif