From 7d71dfa9cf588efb482d2837f44680f32d7565ba Mon Sep 17 00:00:00 2001 From: Henry Le Berre Date: Fri, 26 Jan 2024 11:32:36 -0500 Subject: [PATCH] Handshake: Benchmarking (#316) Co-authored-by: Spencer Bryngelson --- .github/workflows/bench.yml | 52 ++++++++ .github/workflows/phoenix/bench.sh | 11 ++ .github/workflows/test.yml | 4 + toolchain/bench.yaml | 6 + toolchain/main.py | 6 +- toolchain/mfc/args.py | 30 +++-- toolchain/mfc/bench.py | 116 +++++++++++++++++- toolchain/mfc/run/case_dicts.py | 16 +-- toolchain/mfc/run/run.py | 20 +-- toolchain/templates/bridges2.mako | 26 ++-- toolchain/templates/default.mako | 30 ++--- toolchain/templates/include/epilogue.mako | 16 --- .../include/{prologue.mako => helpers.mako} | 56 ++++++++- toolchain/templates/phoenix.mako | 26 ++-- toolchain/templates/summit.mako | 26 ++-- 15 files changed, 321 insertions(+), 120 deletions(-) create mode 100644 .github/workflows/bench.yml create mode 100644 .github/workflows/phoenix/bench.sh create mode 100644 toolchain/bench.yaml delete mode 100644 toolchain/templates/include/epilogue.mako rename toolchain/templates/include/{prologue.mako => helpers.mako} (54%) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml new file mode 100644 index 000000000..c2c31f83e --- /dev/null +++ b/.github/workflows/bench.yml @@ -0,0 +1,52 @@ +name: 'Benchmark' + +on: + pull_request: + +jobs: + self: + name: Georgia Tech | Phoenix (NVHPC) + if: github.repository == 'MFlowCode/MFC' + strategy: + matrix: + device: ['cpu', 'gpu'] + runs-on: + group: phoenix + labels: gt + steps: + - name: Clone - PR + uses: actions/checkout@v3 + with: + path: pr + + - name: Clone - Master + uses: actions/checkout@v3 + with: + repository: MFlowCode/MFC + ref: master + path: master + + - name: Bench (Master v. PR) + run: | + (cd pr && bash .github/workflows/phoenix/submit.sh .github/workflows/phoenix/bench.sh ${{ matrix.device }}) & + (cd master && bash .github/workflows/phoenix/submit.sh .github/workflows/phoenix/bench.sh ${{ matrix.device }}) & + wait %1 && wait %2 + + - name: Generate Comment + run: | + COMMENT_MSG=`./mfc.sh bench_diff master/bench-${{ matrix.device }}.yaml pr/bench-${{ matrix.device }}.yaml` + echo "COMMENT_MSG=\"$COMMENT_MSG\"" >> $GITHUB_ENV + + - name: Post Comment + run: echo "$COMMENT_MSG" + + - name: Archive Logs + uses: actions/upload-artifact@v3 + if: always() + with: + name: logs-${{ matrix.device }} + path: | + pr/bench-${{ matrix.device }}.* + pr/build/benchmarks/* + master/bench-${{ matrix.device }}.* + master/build/benchmarks/* diff --git a/.github/workflows/phoenix/bench.sh b/.github/workflows/phoenix/bench.sh new file mode 100644 index 000000000..78df3dd13 --- /dev/null +++ b/.github/workflows/phoenix/bench.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +n_ranks=4 + +if [ "$job_device" == "gpu" ]; then + n_ranks=$(nvidia-smi -L | wc -l) # number of GPUs on node + gpu_ids=$(seq -s ' ' 0 $(($n_ranks-1))) # 0,1,2,...,gpu_count-1 + device_opts="--gpu -g $gpu_ids" +fi + +./mfc.sh bench -j $(nproc) -o "$job_slug.yaml" -- -c phoenix $device_opts -n $n_ranks diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9cac36163..f3431863a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -117,6 +117,10 @@ jobs: - name: Build & Test run: bash .github/workflows/phoenix/submit.sh .github/workflows/phoenix/test.sh ${{ matrix.device }} + - name: Print Logs + if: always() + run: cat test-${{ matrix.device }}.out + - name: Archive Logs uses: actions/upload-artifact@v3 if: always() diff --git a/toolchain/bench.yaml b/toolchain/bench.yaml new file mode 100644 index 000000000..88f17f2ad --- /dev/null +++ b/toolchain/bench.yaml @@ -0,0 +1,6 @@ +- slug: 3D_shockdroplet + path: examples/3D_shockdroplet/case.py + args: [] +- slug: 3D_turb_mixing + path: examples/3D_turb_mixing/case.py + args: [] diff --git a/toolchain/main.py b/toolchain/main.py index 6c42fd68b..af10904ac 100644 --- a/toolchain/main.py +++ b/toolchain/main.py @@ -47,9 +47,9 @@ def __checks(): def __run(): - {"test": test.test, "run": run.run, "build": build.build, - "clean": build.clean, "bench": bench.bench, "count": count.count, - "packer": packer.packer, "count_diff": count.count_diff + {"test": test.test, "run": run.run, "build": build.build, + "clean": build.clean, "bench": bench.bench, "count": count.count, + "packer": packer.packer, "count_diff": count.count_diff, "bench_diff": bench.diff }[ARG("command")]() diff --git a/toolchain/mfc/args.py b/toolchain/mfc/args.py index 5ac583b6f..6aa43cca4 100644 --- a/toolchain/mfc/args.py +++ b/toolchain/mfc/args.py @@ -18,14 +18,15 @@ def parse(config): ) parsers = parser.add_subparsers(dest="command") - run = parsers.add_parser(name="run", help="Run a case with MFC.", formatter_class=argparse.ArgumentDefaultsHelpFormatter) - test = parsers.add_parser(name="test", help="Run MFC's test suite.", formatter_class=argparse.ArgumentDefaultsHelpFormatter) - build = parsers.add_parser(name="build", help="Build MFC and its dependencies.", formatter_class=argparse.ArgumentDefaultsHelpFormatter) - clean = parsers.add_parser(name="clean", help="Clean build artifacts.", formatter_class=argparse.ArgumentDefaultsHelpFormatter) - bench = parsers.add_parser(name="bench", help="Benchmark MFC (for CI).", formatter_class=argparse.ArgumentDefaultsHelpFormatter) - count = parsers.add_parser(name="count", help="Count LOC in MFC.", formatter_class=argparse.ArgumentDefaultsHelpFormatter) - count_diff = parsers.add_parser(name="count_diff", help="Count LOC in MFC.", formatter_class=argparse.ArgumentDefaultsHelpFormatter) - packer = parsers.add_parser(name="packer", help="Packer utility (pack/unpack/compare)", formatter_class=argparse.ArgumentDefaultsHelpFormatter) + run = parsers.add_parser(name="run", help="Run a case with MFC.", formatter_class=argparse.ArgumentDefaultsHelpFormatter) + test = parsers.add_parser(name="test", help="Run MFC's test suite.", formatter_class=argparse.ArgumentDefaultsHelpFormatter) + build = parsers.add_parser(name="build", help="Build MFC and its dependencies.", formatter_class=argparse.ArgumentDefaultsHelpFormatter) + clean = parsers.add_parser(name="clean", help="Clean build artifacts.", formatter_class=argparse.ArgumentDefaultsHelpFormatter) + bench = parsers.add_parser(name="bench", help="Benchmark MFC (for CI).", formatter_class=argparse.ArgumentDefaultsHelpFormatter) + bench_diff = parsers.add_parser(name="bench_diff", help="Compare MFC Benchmarks (for CI).", formatter_class=argparse.ArgumentDefaultsHelpFormatter) + count = parsers.add_parser(name="count", help="Count LOC in MFC.", formatter_class=argparse.ArgumentDefaultsHelpFormatter) + count_diff = parsers.add_parser(name="count_diff", help="Count LOC in MFC.", formatter_class=argparse.ArgumentDefaultsHelpFormatter) + packer = parsers.add_parser(name="packer", help="Packer utility (pack/unpack/compare)", formatter_class=argparse.ArgumentDefaultsHelpFormatter) packers = packer.add_subparsers(dest="packer") pack = packers.add_parser(name="pack", help="Pack a case into a single file.", formatter_class=argparse.ArgumentDefaultsHelpFormatter) @@ -120,9 +121,17 @@ def add_common_arguments(p, mask = None): run.add_argument("--wait", action="store_true", default=False, help="(Batch) Wait for the job to finish.") run.add_argument("-f", "--flags", metavar="FLAGS", dest="--", nargs=argparse.REMAINDER, type=str, default=[], help="Arguments to forward to the MPI invocation.") run.add_argument("-c", "--computer", metavar="COMPUTER", type=str, default="default", help=f"(Batch) Path to a custom submission file template or one of {format_list_to_string(list(get_baked_templates().keys()))}.") + run.add_argument("-o", "--output-summary", metavar="OUTPUT", type=str, default=None, help="Output file (YAML) for summary.") # === BENCH === - add_common_arguments(bench, "t") + add_common_arguments(bench) + bench.add_argument("-o", "--output", metavar="OUTPUT", default=None, type=str, required="True", help="Path to the YAML output file to write the results to.") + bench.add_argument(metavar="FORWARDED", default=[], dest='--', nargs="*", help="Arguments to forward to the ./mfc.sh run invocations.") + + # === BENCH_DIFF === + add_common_arguments(bench_diff, "t") + bench_diff.add_argument("lhs", metavar="LHS", type=str, help="Path to a benchmark result YAML file.") + bench_diff.add_argument("rhs", metavar="RHS", type=str, help="Path to a benchmark result YAML file.") # === COUNT === add_common_arguments(count, "g") @@ -135,8 +144,7 @@ def add_common_arguments(p, mask = None): # Add default arguments of other subparsers for name, parser in [("run", run), ("test", test), ("build", build), - ("clean", clean), ("bench", bench), ("count", count), - ("count_diff", count_diff)]: + ("clean", clean), ("count", count), ("count_diff", count_diff)]: if args["command"] == name: continue diff --git a/toolchain/mfc/bench.py b/toolchain/mfc/bench.py index 5d4ae85dd..f2429e4c0 100644 --- a/toolchain/mfc/bench.py +++ b/toolchain/mfc/bench.py @@ -1,4 +1,114 @@ -from .common import MFCException +import os, sys, uuid, subprocess, dataclasses -def bench(): - raise MFCException("Benchmarks are currently disabled.") +import rich.table + +from .printer import cons +from .state import ARG, CFG +from .build import get_targets, DEFAULT_TARGETS +from .common import system, MFC_BENCH_FILEPATH, MFC_SUBDIR, format_list_to_string +from .common import file_load_yaml, file_dump_yaml, create_directory + + +@dataclasses.dataclass +class BenchCase: + slug: str + path: str + args: list[str] + + +def bench(targets = None): + if targets is None: + targets = ARG("targets") + + targets = get_targets(targets) + + bench_dirpath = os.path.join(MFC_SUBDIR, "benchmarks", str(uuid.uuid4())[:4]) + create_directory(bench_dirpath) + + cons.print() + cons.print(f"[bold]Benchmarking {format_list_to_string(ARG('targets'), 'magenta')} ([magenta]{os.path.relpath(bench_dirpath)}[/magenta]):[/bold]") + cons.indent() + cons.print() + + CASES = [ BenchCase(**case) for case in file_load_yaml(MFC_BENCH_FILEPATH) ] + + for case in CASES: + case.args = case.args + ARG("--") + case.path = os.path.abspath(case.path) + + results = { + "metadata": { + "invocation": sys.argv[1:], + "lock": dataclasses.asdict(CFG()) + }, + "cases": {}, + } + + for i, case in enumerate(CASES): + summary_filepath = os.path.join(bench_dirpath, f"{case.slug}.yaml") + log_filepath = os.path.join(bench_dirpath, f"{case.slug}.out") + + cons.print(f"{str(i+1).zfill(len(CASES) // 10 + 1)}/{len(CASES)}: {case.slug} @ [bold]{os.path.relpath(case.path)}[/bold]") + cons.indent() + cons.print() + cons.print(f"> Log: [bold]{os.path.relpath(log_filepath)}[/bold]") + cons.print(f"> Summary: [bold]{os.path.relpath(summary_filepath)}[/bold]") + + with open(log_filepath, "w") as log_file: + system( + ["./mfc.sh", "run", case.path, "--case-optimization"] + + ["--targets"] + [t.name for t in targets] + + ["--output-summary", summary_filepath] + + case.args, + stdout=log_file, + stderr=subprocess.STDOUT) + + results["cases"][case.slug] = { + "description": dataclasses.asdict(case), + "output_summary": file_load_yaml(summary_filepath), + } + + file_dump_yaml(ARG("output"), results) + + cons.print(f"Wrote results to [bold magenta]{os.path.relpath(ARG('output'))}[/bold magenta].") + + cons.unindent() + + +def diff(): + lhs, rhs = file_load_yaml(ARG("lhs")), file_load_yaml(ARG("rhs")) + + cons.print(f"[bold]Comparing Bencharks: [magenta]{os.path.relpath(ARG('lhs'))}[/magenta] is x times slower than [magenta]{os.path.relpath(ARG('rhs'))}[/magenta].[/bold]") + + if lhs["metadata"] != rhs["metadata"]: + cons.print(f"[bold yellow]Warning[/bold yellow]: Metadata of lhs and rhs are not equal.") + quit(1) + + slugs = set(lhs["cases"].keys()) & set(rhs["cases"].keys()) + if len(slugs) not in [len(lhs["cases"]), len(rhs["cases"])]: + cons.print(f"[bold yellow]Warning[/bold yellow]: Cases of lhs and rhs are not equal.[/bold yellow]") + cons.print(f"[bold yellow]lhs: {set(lhs['cases'].keys()) - slugs}[/bold yellow]") + cons.print(f"[bold yellow]rhs: {set(rhs['cases'].keys()) - slugs}[/bold yellow]") + cons.print(f"[bold yellow]Using intersection: {slugs}[/bold yellow]") + + table = rich.table.Table(show_header=True, box=rich.table.box.SIMPLE) + table.add_column("[bold]Case[/bold]", justify="left") + table.add_column("[bold]Pre Process[/bold]", justify="right") + table.add_column("[bold]Simulation[/bold]", justify="right") + table.add_column("[bold]Post Process[/bold]", justify="right") + + for slug in slugs: + lhs_summary = lhs["cases"][slug]["output_summary"] + rhs_summary = rhs["cases"][slug]["output_summary"] + + speedups = ['N/A', 'N/A', 'N/A'] + + for i, target in enumerate(sorted(DEFAULT_TARGETS, key=lambda t: t.runOrder)): + if target.name not in lhs_summary or target.name not in rhs_summary: + continue + + speedups[i] = f"{lhs_summary[target.name] / rhs_summary[target.name]:.2f}x" + + table.add_row(f"[magenta]{slug}[/magenta]", *speedups) + + cons.raw.print(table) diff --git a/toolchain/mfc/run/case_dicts.py b/toolchain/mfc/run/case_dicts.py index 7d9c465f0..a12cc858f 100644 --- a/toolchain/mfc/run/case_dicts.py +++ b/toolchain/mfc/run/case_dicts.py @@ -1,4 +1,3 @@ -from .. import common from ..state import ARG @@ -150,16 +149,11 @@ def get_input_dict_keys(target_name: str) -> list: - result = None - if target_name == "pre_process": - result = PRE_PROCESS.copy() - if target_name == "simulation": - result = SIMULATION.copy() - if target_name == "post_process": - result = POST_PROCESS.copy() - - if result is None: - raise common.MFCException(f"[INPUT DICTS] Target {target_name} doesn't have an input dict.") + result = { + "pre_process" : PRE_PROCESS, + "simulation" : SIMULATION, + "post_process" : POST_PROCESS + }.get(target_name, {}).copy() if not ARG("case_optimization") or target_name != "simulation": return result diff --git a/toolchain/mfc/run/run.py b/toolchain/mfc/run/run.py index 9b81121d8..575298c94 100644 --- a/toolchain/mfc/run/run.py +++ b/toolchain/mfc/run/run.py @@ -1,16 +1,16 @@ -import re, os, typing +import re, os, sys, typing, dataclasses from glob import glob from mako.lookup import TemplateLookup from mako.template import Template -from ..build import get_targets, build +from ..build import get_targets, build, REQUIRED_TARGETS from ..printer import cons -from ..state import ARG, ARGS +from ..state import ARG, ARGS, CFG from ..common import MFCException, isspace, file_read, does_command_exist from ..common import MFC_TEMPLATEDIR, file_write, system, MFC_ROOTDIR -from ..common import format_list_to_string +from ..common import format_list_to_string, file_dump_yaml from . import queues, input @@ -84,12 +84,11 @@ def __generate_job_script(targets): env['CUDA_VISIBLE_DEVICES'] = ','.join([str(_) for _ in ARG('gpus')]) content = __get_template().render( - **ARGS(), + **{**ARGS(), 'targets': targets}, ARG=ARG, env=env, - rootdir=MFC_ROOTDIR, + MFC_ROOTDIR=MFC_ROOTDIR, qsystem=queues.get_system(), - binpaths=[target.get_install_binpath() for target in targets], profiler=__profiler_prepend(), ) @@ -119,7 +118,7 @@ def __execute_job_script(qsystem: queues.QueueSystem): def run(targets = None): - targets = get_targets(targets or ARG("targets")) + targets = get_targets(list(REQUIRED_TARGETS) + (targets or ARG("targets"))) build(targets) @@ -134,4 +133,9 @@ def run(targets = None): __generate_input_files(targets) if not ARG("dry_run"): + if ARG("output_summary") is not None: + file_dump_yaml(ARG("output_summary"), { + "invocation": sys.argv[1:], + "lock": dataclasses.asdict(CFG()) + }) __execute_job_script(qsystem) diff --git a/toolchain/templates/bridges2.mako b/toolchain/templates/bridges2.mako index 34d718716..12a671e91 100644 --- a/toolchain/templates/bridges2.mako +++ b/toolchain/templates/bridges2.mako @@ -1,5 +1,7 @@ #!/usr/bin/env bash +<%namespace name="helpers" file="helpers.mako"/> + % if engine == 'batch': #SBATCH --nodes=${nodes} #SBATCH --ntasks-per-node=${tasks_per_node} @@ -25,37 +27,29 @@ % endif % endif -<%include file="prologue.mako"/> +${helpers.template_prologue()} ok ":) Loading modules:\n" -cd "${rootdir}" +cd "${MFC_ROOTDIR}" . ./mfc.sh load -c b -m ${'g' if gpu else 'c'} cd - > /dev/null echo -% for binpath in binpaths: - ok -e ":) Running ${binpath.split('/')[-1]}:\n" +% for target in targets: + ${helpers.run_prologue(target)} % if not mpi: - ${' '.join([f"'{x}'" for x in profiler ])} "${binpath}" + ${' '.join([f"'{x}'" for x in profiler ])} "${target.get_install_binpath()}" % else: ${' '.join([f"'{x}'" for x in profiler ])} \ mpirun -np ${nodes*tasks_per_node} \ ${' '.join([f"'{x}'" for x in ARG('--') ])} \ - "${binpath}" + "${target.get_install_binpath()}" % endif - % if engine == 'interactive': - code=$? - if [ $code -ne 0 ]; then - echo - error ":( $MAGENTA${binpath}$COLOR_RESET failed with exit code $MAGENTA$code$COLOR_RESET." - echo - exit 1 - fi - % endif + ${helpers.run_epilogue(target)} echo % endfor -<%include file="epilogue.mako"/> +${helpers.template_epilogue()} diff --git a/toolchain/templates/default.mako b/toolchain/templates/default.mako index 9917e0517..bf1b6948b 100644 --- a/toolchain/templates/default.mako +++ b/toolchain/templates/default.mako @@ -1,17 +1,17 @@ #!/usr/bin/env bash -. "${rootdir}/toolchain/util.sh" +<%namespace name="helpers" file="helpers.mako"/> + +${helpers.template_prologue()} % if engine == 'batch': error "The$MAGENTA default$COLOR_RESET template does not support batch jobs. Please use a different template via the $MAGENTA--computer$COLOR_RESET option.\n" exit 1 % endif -<%include file="prologue.mako"/> - warn "This is the$MAGENTA default$COLOR_RESET template." warn "It is not intended to support all systems and execution engines." -warn "Please use a different template via the $MAGENTA--computer$COLOR_RESET option." +warn "Consider using a different template via the $MAGENTA--computer$COLOR_RESET option if you encounter problems." % if mpi: # Find a suitable MPI launcher and store it in the variable "binary". @@ -29,11 +29,11 @@ warn "Please use a different template via the $MAGENTA--computer$COLOR_RESET opt fi % endif -% for binpath in binpaths: - ok ":) Running $MAGENTA${binpath}$COLOR_RESET:\n" +% for target in targets: + ${helpers.run_prologue(target)} % if not mpi: - ${' '.join([f"'{x}'" for x in profiler ])} "${binpath}" + ${' '.join([f"'{x}'" for x in profiler ])} "${target.get_install_binpath()}" % else: if [ "$binary" == "jsrun" ]; then ${' '.join([f"'{x}'" for x in profiler ])} \ @@ -42,29 +42,23 @@ warn "Please use a different template via the $MAGENTA--computer$COLOR_RESET opt --gpu_per_rs ${1 if gpu else 0} \ --tasks_per_rs 1 \ ${' '.join([f"'{x}'" for x in ARG('--') ])} \ - "${binpath}" + "${target.get_install_binpath()}" elif [ "$binary" == "srun" ]; then ${' '.join([f"'{x}'" for x in profiler ])} \ srun --ntasks-per-node ${tasks_per_node} \ ${' '.join([f"'{x}'" for x in ARG('--') ])} \ - "${binpath}" + "${target.get_install_binpath()}" elif [ "$binary" == "mpirun" ] || [ "$binary" == "mpiexec" ]; then ${' '.join([f"'{x}'" for x in profiler ])} \ $binary -np ${nodes*tasks_per_node} \ ${' '.join([f"'{x}'" for x in ARG('--') ])} \ - "${binpath}" + "${target.get_install_binpath()}" fi % endif - code=$? - if [ $code -ne 0 ]; then - echo - error ":( $MAGENTA${binpath}$COLOR_RESET failed with exit code $MAGENTA$code$COLOR_RESET." - echo - exit 1 - fi + ${helpers.run_epilogue(target)} echo % endfor -<%include file="epilogue.mako"/> +${helpers.template_epilogue()} diff --git a/toolchain/templates/include/epilogue.mako b/toolchain/templates/include/epilogue.mako deleted file mode 100644 index 183b6b8f5..000000000 --- a/toolchain/templates/include/epilogue.mako +++ /dev/null @@ -1,16 +0,0 @@ -#> -#> The MFC epilogue stops the timer and prints the execution summary. It also -#> performs some cleanup and housekeeping tasks before exiting. -#> - -code=$? - -t_stop="$(date +%s)" - -printf "$TABLE_HEADER" -printf "$TABLE_TITLE_FORMAT" "Finished" "$MAGENTA${name}$COLOR_RESET:" -printf "$TABLE_FORMAT_LINE" "Total-time:" "$(expr $t_stop - $t_start)s" "Exit Code:" "$code" -printf "$TABLE_FORMAT_LINE" "End-time:" "$(date +%T)" "End-date:" "$(date +%T)" -printf "$TABLE_FOOTER" - -exit $code \ No newline at end of file diff --git a/toolchain/templates/include/prologue.mako b/toolchain/templates/include/helpers.mako similarity index 54% rename from toolchain/templates/include/prologue.mako rename to toolchain/templates/include/helpers.mako index aa9962042..262fc5917 100644 --- a/toolchain/templates/include/prologue.mako +++ b/toolchain/templates/include/helpers.mako @@ -1,3 +1,4 @@ +<%def name="template_prologue()"> #> #> The MFC prologue prints a summary of the running job and starts a timer. #> @@ -6,7 +7,7 @@ import os %>\ -. "${rootdir}/toolchain/util.sh" +. "${MFC_ROOTDIR}/toolchain/util.sh" TABLE_FORMAT_LINE="| * %-14s $MAGENTA%-35s$COLOR_RESET * %-14s $MAGENTA%-35s$COLOR_RESET |\\n" TABLE_HEADER="+-----------------------------------------------------------------------------------------------------------+ \\n" @@ -32,6 +33,57 @@ printf "$TABLE_FOOTER\\n" export ${key}='${value}' % endfor +t_start=$(date +%s) + + +<%def name="template_epilogue()"> +#> +#> The MFC epilogue stops the timer and prints the execution summary. It also +#> performs some cleanup and housekeeping tasks before exiting. +#> + +code=$? + +t_stop="$(date +%s)" + +printf "$TABLE_HEADER" +printf "$TABLE_TITLE_FORMAT" "Finished ${name}:" +printf "$TABLE_FORMAT_LINE" "Total-time:" "$(expr $t_stop - $t_start)s" "Exit Code:" "$code" +printf "$TABLE_FORMAT_LINE" "End-time:" "$(date +%T)" "End-date:" "$(date +%T)" +printf "$TABLE_FOOTER" + +exit $code + + +<%def name="run_prologue(target)"> +ok ":) Running$MAGENTA ${target.name}$COLOR_RESET:\n" + cd "${os.path.dirname(input)}" -t_start=$(date +%s) \ No newline at end of file +t_${target.name}_start=$(date +%s) + + +<%def name="run_epilogue(target)"> +code=$? + +t_${target.name}_stop=$(date +%s) + +if [ $code -ne 0 ]; then + echo + error ":( $MAGENTA${target.get_install_binpath()}$COLOR_RESET failed with exit code $MAGENTA$code$COLOR_RESET." + echo + exit 1 +fi + +% if output_summary: + +cd "${MFC_ROOTDIR}" + +cat >>"${output_summary}" < /dev/null + +% endif + \ No newline at end of file diff --git a/toolchain/templates/phoenix.mako b/toolchain/templates/phoenix.mako index 80c70f97e..e40e32fd7 100644 --- a/toolchain/templates/phoenix.mako +++ b/toolchain/templates/phoenix.mako @@ -1,5 +1,7 @@ #!/usr/bin/env bash +<%namespace name="helpers" file="helpers.mako"/> + % if engine == 'batch': #SBATCH --nodes=${nodes} #SBATCH --ntasks-per-node=${tasks_per_node} @@ -25,38 +27,30 @@ % endif % endif -<%include file="prologue.mako"/> +${helpers.template_prologue()} ok ":) Loading modules:\n" -cd "${rootdir}" +cd "${MFC_ROOTDIR}" . ./mfc.sh load -c p -m ${'g' if gpu else 'c'} cd - > /dev/null echo -% for binpath in binpaths: - ok ":) Running ${binpath.split('/')[-1]}:\n" +% for target in targets: + ${helpers.run_prologue(target)} % if not mpi: - ${' '.join([f"'{x}'" for x in profiler ])} "${binpath}" + ${' '.join([f"'{x}'" for x in profiler ])} "${target.get_install_binpath()}" % else: ${' '.join([f"'{x}'" for x in profiler ])} \ mpirun -np ${nodes*tasks_per_node} \ --bind-to none \ ${' '.join([f"'{x}'" for x in ARG('--') ])} \ - "${binpath}" + "${target.get_install_binpath()}" % endif - % if engine == 'interactive': - code=$? - if [ $code -ne 0 ]; then - echo - error ":( $MAGENTA${binpath}$COLOR_RESET failed with exit code $MAGENTA$code$COLOR_RESET." - echo - exit 1 - fi - % endif + ${helpers.run_epilogue(target)} echo % endfor -<%include file="epilogue.mako"/> +${helpers.template_epilogue()} diff --git a/toolchain/templates/summit.mako b/toolchain/templates/summit.mako index a9a888c41..056ddb3d4 100644 --- a/toolchain/templates/summit.mako +++ b/toolchain/templates/summit.mako @@ -1,5 +1,7 @@ #!/usr/bin/env bash +<%namespace name="helpers" file="helpers.mako"/> + % if engine == 'batch': #BSUB -J {{{name}}} #BSUB -nnodes {{{nodes}}} @@ -10,19 +12,19 @@ % endif % endif -<%include file="prologue.mako"/> +${helpers.template_prologue()} ok ":) Loading modules:\n" -cd "${rootdir}" +cd "${MFC_ROOTDIR}" . ./mfc.sh load -c s -m ${'g' if gpu else 'c'} cd - > /dev/null echo -% for binpath in binpaths: - ok ":) Running ${binpath}:\n" +% for target in targets: + ${helpers.run_prologue(target)} % if not mpi: - ${' '.join([f"'{x}'" for x in profiler ])} "${binpath}" + ${' '.join([f"'{x}'" for x in profiler ])} "${target.get_install_binpath()}" % else: ${' '.join([f"'{x}'" for x in profiler ])} \ jsrun \ @@ -32,20 +34,12 @@ echo --gpu_per_rs ${1 if gpu else 0} \ --tasks_per_rs 1 \ ${' '.join([f"'{x}'" for x in ARG('--') ])} \ - "${binpath}" + "${target.get_install_binpath()}" % endif - % if engine == 'interactive': - code=$? - if [ $code -ne 0 ]; then - echo - error ":( $MAGENTA${binpath}$COLOR_RESET failed with exit code $MAGENTA$code$COLOR_RESET." - echo - exit 1 - fi - % endif + ${helpers.run_epilogue(target)} echo % endfor -<%include file="epilogue.mako"/> +${helpers.template_epilogue()}