Skip to content

Commit 21fffb2

Browse files
henryleberrehenryleberre
committed
#49: GitHub CI Benchmarking
1 parent 371c51a commit 21fffb2

File tree

13 files changed

+245
-107
lines changed

13 files changed

+245
-107
lines changed

‎.github/workflows/bench.yml

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
name: 'Benchmark'
2+
3+
on:
4+
push:
5+
paths:
6+
- '**.f90'
7+
- '**.fpp'
8+
- '**.py'
9+
- '**.yml'
10+
- 'mfc.sh'
11+
- 'CMakeLists.txt'
12+
- 'requirements.txt'
13+
14+
pull_request:
15+
16+
jobs:
17+
self:
18+
name: Georgia Tech | Phoenix (NVHPC)
19+
if: github.repository == 'MFlowCode/MFC'
20+
strategy:
21+
matrix:
22+
device: ['cpu', 'gpu']
23+
runs-on:
24+
group: phoenix
25+
labels: self-hosted
26+
steps:
27+
- name: Clone
28+
uses: actions/checkout@v3
29+
30+
- name: Bench - PR
31+
run: |
32+
bash .github/workflows/phoenix/submit.sh .github/workflows/phoenix/bench.sh ${{ matrix.device }}
33+
mv bench-${{ matrix.device }}.out bench-${{ matrix.device }}-pr.out
34+
35+
- name: Bench - Master
36+
run: |
37+
git remote add upstream https://github.com/MFlowCode/MFC.git
38+
git fetch upstream
39+
git checkout upstream/master
40+
bash .github/workflows/phoenix/submit.sh .github/workflows/phoenix/bench.sh ${{ matrix.device }}
41+
mv bench-${{ matrix.device }}.out bench-${{ matrix.device }}-master.out
42+
43+
- name: Generate Comment
44+
run: |
45+
export "BENCH_COMMENT="$(python3 .github/workflows/phoenix/compare.py bench-${{ matrix.device }}-master.out bench-${{ matrix.device }}-pr.out)"" >> $GITHUB_ENV
46+
47+
- name: Post Comment
48+
uses: actions/github-script@v6
49+
with:
50+
github-token: ${{ secrets.GITHUB_TOKEN }}
51+
script: |
52+
github.issues.createComment({
53+
issue_number: context.issue.number,
54+
owner: context.repo.owner,
55+
repo: context.repo.repo,
56+
body: process.env.BENCH_COMMENT
57+
})
58+
59+
- name: Print
60+
if: always()
61+
run: |
62+
cat bench-${{ matrix.device }}-master.out || true
63+
cat bench-${{ matrix.device }}-pr.out || true

‎.github/workflows/docker.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ jobs:
2222
password: ${{ secrets.DOCKER_PASSWORD }}
2323

2424
- name: Build & Publish thereto
25-
uses: docker/build-push-action@v4
25+
uses: docker/build-push-action@v3
2626
with:
2727
file: toolchain/Dockerfile
2828
push: true

‎.github/workflows/phoenix/bench.sh

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
#!/bin/bash
2+
3+
./mfc.sh bench "$job_slug.yaml" -j $(nproc) -b mpirun

‎.github/workflows/phoenix/compare.py

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
#!/usr/bin/env python3
2+
3+
import argparse
4+
5+
import yaml
6+
7+
parser = argparse.ArgumentParser()
8+
parser.add_argument('master', metavar="MASTER", type=str)
9+
parser.add_argument('pr', metavar="PR", type=str)
10+
11+
args = parser.parse_args()
12+
13+
def load_cases(filepath):
14+
return { case["name"]: case for case in yaml.safe_load(open(filepath))["cases"] }
15+
16+
master, pr = load_cases(args.master), load_cases(args.pr)
17+
18+
master_keys = set(master.keys())
19+
pr_keys = set(pr.keys())
20+
21+
missing_cases = master_keys.symmetric_difference(pr_keys)
22+
23+
if len(missing_cases) > 0:
24+
print("**Warning:** The following cases are **missing** from master or this PR:\n")
25+
26+
for case in missing_cases:
27+
print(f" - {case}.")
28+
29+
print("")
30+
31+
speedups = {}
32+
33+
for case in master_keys.intersection(pr_keys):
34+
speedups[case] = {
35+
"pre_proess": pr[case]["pre_process"] / master[case]["pre_process"],
36+
"simulation": pr[case]["simulation"] / master[case]["simulation"],
37+
}
38+
39+
avg_speedup = sum([ speedups[case]["simulation"] for case in speedups ]) / len(speedups)
40+
41+
print(f"""\
42+
**[Benchmark Results]** Compared to Master, this PR's `simulation` is on average **~{avg_speedup:0.2f}x faster**.
43+
44+
| **Case** | **Master** | **PR** | **Speedup** |
45+
| -------- | ---------- | ------ | ----------- |\
46+
""")
47+
48+
for case in sorted(speedups.keys()):
49+
speedup = speedups[case]
50+
51+
print(f"| {case} | {master[case]['simulation']:0.2f}s | {pr[case]['simulation']:0.2f}s | {speedups[case]['simulation']:0.2f}x |")

‎.github/workflows/phoenix/submit.sh

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
#!/bin/bash
2+
3+
usage() {
4+
echo "Usage: $0 [script.sh] [cpu|gpu]"
5+
}
6+
7+
if [ ! -z "$1" ]; then
8+
sbatch_script_contents=`cat $1`
9+
else
10+
usage
11+
exit 1
12+
fi
13+
14+
sbatch_cpu_opts="\
15+
#SBATCH --ntasks-per-node=12 # Number of cores per node required
16+
#SBATCH --mem-per-cpu=2G # Memory per core\
17+
"
18+
19+
sbatch_gpu_opts="\
20+
#SBATCH -CV100-16GB
21+
#SBATCH -G2\
22+
"
23+
24+
if [ "$2" == "cpu" ]; then
25+
sbatch_device_opts="$sbatch_cpu_opts"
26+
elif [ "$2" == "gpu" ]; then
27+
sbatch_device_opts="$sbatch_gpu_opts"
28+
else
29+
usage
30+
exit 1
31+
fi
32+
33+
job_slug="`basename "$1" | sed 's/\.sh$//' | sed 's/[^a-zA-Z0-9]/-/g'`-$2"
34+
35+
sbatch <<EOT
36+
#!/bin/bash
37+
#SBATCH -Jshb-$job_slug # Job name
38+
#SBATCH --account=gts-sbryngelson3 # charge account
39+
#SBATCH -N1 # Number of nodes required
40+
$sbatch_device_opts
41+
#SBATCH -t 04:00:00 # Duration of the job (Ex: 15 mins)
42+
#SBATCH -q embers # QOS Name
43+
#SBATCH -o$job_slug.out # Combined output and error messages file
44+
#SBATCH -W # Do not exit until the submitted job terminates.
45+
46+
set -x
47+
48+
cd "\$SLURM_SUBMIT_DIR"
49+
echo "Running in $(pwd):"
50+
51+
job_slug="$job_slug"
52+
job_device="$2"
53+
54+
. ./mfc.sh load -c p -m $2
55+
56+
$sbatch_script_contents
57+
58+
EOT

‎.github/workflows/phoenix/test.sh

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#!/bin/bash
2+
3+
if [ "$job_device" == "gpu" ]; then
4+
gpu_count=$(nvidia-smi -L | wc -l) # number of GPUs on node
5+
gpu_ids=$(seq -s ' ' 0 $(($gpu_count-1))) # 0,1,2,...,gpu_count-1
6+
device_opts="--gpu -g $gpu_ids"
7+
fi
8+
9+
./mfc.sh test -a -b mpirun -j $(nproc) $device_opts

‎.github/workflows/ci.yml renamed to ‎.github/workflows/test.yml

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -120,13 +120,8 @@ jobs:
120120
./mfc.sh build -j 2 $(if [ '${{ matrix.device }}' == 'gpu' ]; then echo '--gpu'; fi)
121121
122122
- name: Test
123-
run: |
124-
. ./mfc.sh load -c p -m gpu
125-
mv misc/run-phoenix-release-${{ matrix.device }}.sh ./
126-
sbatch run-phoenix-release-${{ matrix.device }}.sh
123+
run: bash .github/workflows/phoenix/submit.sh .github/workflows/phoenix/test.sh ${{ matrix.device }}
127124

128125
- name: Print
129126
if: always()
130-
run: |
131-
cat test.out
132-
127+
run: cat test-${{ matrix.device }}.out

‎misc/run-phoenix-release-cpu.sh

Lines changed: 0 additions & 16 deletions
This file was deleted.

‎misc/run-phoenix-release-gpu.sh

Lines changed: 0 additions & 24 deletions
This file was deleted.

‎toolchain/bench.yaml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
- name: 1D_bubblescreen
2+
path: examples/1D_bubblescreen/case.py
3+
args: []
4+
5+
- name: 1D_kapilashocktube
6+
path: examples/1D_kapilashocktube/case.py
7+
args: []

‎toolchain/mfc/args.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,9 @@ def add_common_arguments(p, mask = None):
122122
run.add_argument("--wait", action="store_true", default=False, help="(Batch) Wait for the job to finish.")
123123

124124
# === BENCH ===
125-
add_common_arguments(bench, "t")
125+
add_common_arguments(bench, "tjgn")
126+
bench.add_argument("output", metavar="OUTPUT", default=None, type=str, help="Path to the YAML output file to write the results to.")
127+
bench.add_argument(metavar="FORWARDED", default=[], dest='forwarded', nargs=argparse.REMAINDER, help="Arguments to forward to the ./mfc.sh run invocations.")
126128

127129
# === COUNT ===
128130
add_common_arguments(count, "g")

‎toolchain/mfc/bench.py

Lines changed: 43 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -1,62 +1,53 @@
1-
import os, json, time, typing, datetime, subprocess
2-
3-
import rich.table
1+
import sys, time, subprocess, dataclasses
42

53
from .printer import cons
6-
from .state import ARG
4+
from .state import ARG, CFG
75
from .build import PRE_PROCESS, SIMULATION, build_targets
8-
from .common import system, MFC_SUBDIR
6+
from .common import system, MFC_BENCH_FILEPATH, file_load_yaml, file_dump_yaml
97
from . import sched
108

9+
10+
@dataclasses.dataclass
11+
class BenchCase:
12+
name: str
13+
path: str
14+
args: list[str]
15+
16+
1117
def bench():
12-
build_targets([PRE_PROCESS, SIMULATION])
13-
18+
cons.print()
1419
cons.print("[bold]Benchmarking [magenta]simulation[/magenta]:[/bold]")
1520
cons.indent()
16-
17-
CASES = ["1D_bubblescreen", "1D_exercise_WENO", "1D_kapilashocktube"]
18-
RESULTS = []
19-
20-
table = rich.table.Table(show_lines=False, show_edge=False)
21-
table.add_column("Case")
22-
table.add_column("(Simulation) Runtime (s)")
23-
24-
def __worker(case: str, devices: typing.Set[int]):
25-
nonlocal RESULTS
26-
27-
system(["./mfc.sh", "run", f"examples/{case}/case.py", "--no-build", "-t", "pre_process"], stdout=subprocess.DEVNULL)
28-
start = time.monotonic()
29-
system(["./mfc.sh", "run", f"examples/{case}/case.py", "--no-build", "-t", "simulation"], stdout=subprocess.DEVNULL)
30-
end = time.monotonic()
31-
runtime = datetime.timedelta(seconds=end - start).total_seconds()
32-
33-
RESULTS.append({
34-
"name": f"Simulation: {case}",
35-
"unit": "seconds",
36-
"value": runtime
37-
})
38-
39-
table.add_row(case, str(runtime))
40-
41-
tasks: typing.List[sched.Task] = [
42-
sched.Task(1, __worker, [ case ], 1) for case in CASES
43-
]
44-
4521
cons.print()
46-
nThreads = min(ARG('jobs'), len(ARG('gpus'))) if ARG("gpu") else ARG('jobs')
47-
if ARG('case_optimization'):
48-
nThreads = 1
4922

50-
sched.sched(tasks, nThreads, ARG("gpus"))
51-
cons.print()
52-
cons.unindent()
53-
cons.print("[bold]Benchmark Results:[/bold]")
54-
cons.print()
55-
cons.raw.print(table)
56-
cons.print()
57-
58-
filepath = os.path.join(MFC_SUBDIR, "bench.json")
59-
with open(filepath, "w") as f:
60-
json.dump(RESULTS, f)
61-
62-
cons.print(f"[bold green]✓[/bold green] Saved results to [magenta]{filepath}[/magenta].")
23+
CASES = [ BenchCase(**case) for case in file_load_yaml(MFC_BENCH_FILEPATH) ]
24+
25+
for case in CASES:
26+
case.args = case.args + ARG("forwarded")
27+
28+
cons.print(f"Found [magenta]{len(CASES)}[/magenta] cases.")
29+
30+
results = {
31+
"metadata": {
32+
"invocation": sys.argv[1:],
33+
"lock": dataclasses.asdict(CFG())
34+
},
35+
"cases": [],
36+
}
37+
38+
for i, case in enumerate(CASES):
39+
cons.print(f"{str(i+1).zfill(len(CASES) // 10 + 1)}/{len(CASES)}: {case.name} @ [bold]{case.path}[/bold]")
40+
system(["./mfc.sh", "build", "--targets", "pre_process", "simulation", "--case-optimization", "--input", case.path], stdout=subprocess.DEVNULL)
41+
42+
case_results = dataclasses.asdict(case)
43+
44+
for target in [PRE_PROCESS, SIMULATION]:
45+
start = time.time()
46+
system(["./mfc.sh", "run", case.path, "--targets", target.name, "--case-optimization", *case.args], stdout=subprocess.DEVNULL)
47+
case_results[target.name] = time.time() - start
48+
49+
results["cases"].append(case_results)
50+
51+
file_dump_yaml(ARG("output"), results)
52+
53+
cons.unindent()

0 commit comments

Comments
 (0)