Skip to content

Commit ccfb0c9

Browse files
henryleberrehenryleberre
committed
#49: GitHub CI Benchmarking
1 parent d494833 commit ccfb0c9

File tree

15 files changed

+262
-115
lines changed

15 files changed

+262
-115
lines changed

‎.github/workflows/bench.yml

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
name: 'Benchmark'
2+
3+
on:
4+
pull_request_target:
5+
types: [opened, synchronize, reopened]
6+
7+
jobs:
8+
self:
9+
name: Georgia Tech | Phoenix (NVHPC)
10+
if: github.repository == 'MFlowCode/MFC'
11+
strategy:
12+
matrix:
13+
device: ['cpu', 'gpu']
14+
runs-on:
15+
group: phoenix
16+
labels: self-hosted
17+
steps:
18+
- name: Clone - PR
19+
uses: actions/checkout@v3
20+
21+
- name: Bench - PR
22+
run: |
23+
bash .github/workflows/phoenix/submit.sh .github/workflows/phoenix/bench.sh ${{ matrix.device }}
24+
mv bench-${{ matrix.device }}.out ~/bench-${{ matrix.device }}-pr.out
25+
mv bench-${{ matrix.device }}.yaml ~/bench-${{ matrix.device }}-pr.yaml
26+
27+
- name: Clone - Master
28+
uses: actions/checkout@v3
29+
with:
30+
repository: henryleberre/MFC
31+
ref: master
32+
33+
- name: Bench - Master
34+
run: |
35+
bash .github/workflows/phoenix/submit.sh .github/workflows/phoenix/bench.sh ${{ matrix.device }}
36+
mv bench-${{ matrix.device }}.out ~/bench-${{ matrix.device }}-master.out
37+
mv bench-${{ matrix.device }}.yaml ~/bench-${{ matrix.device }}-master.yaml
38+
39+
- name: Generate Comment
40+
run: |
41+
COMMENT_MSG=`python3 .github/workflows/phoenix/compare.py ~/bench-${{ matrix.device }}-master.yaml ~/bench-${{ matrix.device }}-pr.yaml`
42+
echo "COMMENT_MSG=\"$COMMENT_MSG\"" >> $GITHUB_ENV
43+
44+
- name: Post Comment
45+
uses: thollander/actions-comment-pull-request@v2
46+
with:
47+
message: ${{ env.COMMENT_MSG }}
48+
49+
- name: Archive Logs
50+
uses: actions/upload-artifact@v3
51+
if: always()
52+
with:
53+
name: logs-${{ matrix.device }}
54+
path: |
55+
~/bench-${{ matrix.device }}-*

‎.github/workflows/docker.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ jobs:
2222
password: ${{ secrets.DOCKER_PASSWORD }}
2323

2424
- name: Build & Publish thereto
25-
uses: docker/build-push-action@v4
25+
uses: docker/build-push-action@v3
2626
with:
2727
file: toolchain/Dockerfile
2828
push: true

‎.github/workflows/docs.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ on:
88
workflow_dispatch:
99

1010
jobs:
11+
if: secrets.DOC_PUSH_URL
1112
docs:
1213
name: Build & Publish
1314
runs-on: ubuntu-latest

‎.github/workflows/phoenix/bench.sh

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
#!/bin/bash
2+
3+
n_ranks=12
4+
5+
if [ "$job_device" == "gpu" ]; then
6+
n_ranks=$(nvidia-smi -L | wc -l) # number of GPUs on node
7+
gpu_ids=$(seq -s ' ' 0 $(($n_ranks-1))) # 0,1,2,...,gpu_count-1
8+
device_opts="--gpu -g $gpu_ids"
9+
fi
10+
11+
./mfc.sh bench "$job_slug.yaml" -j $(nproc) -b mpirun $device_opts -n $n_ranks

‎.github/workflows/phoenix/compare.py

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
#!/usr/bin/env python3
2+
3+
import argparse
4+
5+
import yaml
6+
7+
parser = argparse.ArgumentParser()
8+
parser.add_argument('master', metavar="MASTER", type=str)
9+
parser.add_argument('pr', metavar="PR", type=str)
10+
11+
args = parser.parse_args()
12+
13+
def load_cases(filepath):
14+
return { case["name"]: case for case in yaml.safe_load(open(filepath))["cases"] }
15+
16+
master, pr = load_cases(args.master), load_cases(args.pr)
17+
18+
master_keys = set(master.keys())
19+
pr_keys = set(pr.keys())
20+
21+
missing_cases = master_keys.symmetric_difference(pr_keys)
22+
23+
if len(missing_cases) > 0:
24+
print("**Warning:** The following cases are **missing** from master or this PR:\n")
25+
26+
for case in missing_cases:
27+
print(f" - {case}.")
28+
29+
print("")
30+
31+
speedups = {}
32+
33+
for case in master_keys.intersection(pr_keys):
34+
speedups[case] = {
35+
"pre_proess": pr[case]["pre_process"] / master[case]["pre_process"],
36+
"simulation": pr[case]["simulation"] / master[case]["simulation"],
37+
}
38+
39+
avg_speedup = sum([ speedups[case]["simulation"] for case in speedups ]) / len(speedups)
40+
41+
print(f"""\
42+
**[Benchmark Results]** Compared to Master, this PR's `simulation` is on average **~{avg_speedup:0.2f}x faster**.
43+
44+
| **Case** | **Master** | **PR** | **Speedup** |
45+
| -------- | ---------- | ------ | ----------- |\
46+
""")
47+
48+
for case in sorted(speedups.keys()):
49+
speedup = speedups[case]
50+
51+
print(f"| {case} | {master[case]['simulation']:0.2f}s | {pr[case]['simulation']:0.2f}s | {speedups[case]['simulation']:0.2f}x |")

‎.github/workflows/phoenix/submit.sh

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
#!/bin/bash
2+
3+
set -e
4+
5+
usage() {
6+
echo "Usage: $0 [script.sh] [cpu|gpu]"
7+
}
8+
9+
if [ ! -z "$1" ]; then
10+
sbatch_script_contents=`cat $1`
11+
else
12+
usage
13+
exit 1
14+
fi
15+
16+
sbatch_cpu_opts="\
17+
#SBATCH -p cpu-small # partition
18+
#SBATCH --ntasks-per-node=24 # Number of cores per node required
19+
#SBATCH --mem-per-cpu=2G # Memory per core\
20+
"
21+
22+
sbatch_gpu_opts="\
23+
#SBATCH -CV100-16GB
24+
#SBATCH -G2\
25+
"
26+
27+
if [ "$2" == "cpu" ]; then
28+
sbatch_device_opts="$sbatch_cpu_opts"
29+
elif [ "$2" == "gpu" ]; then
30+
sbatch_device_opts="$sbatch_gpu_opts"
31+
else
32+
usage
33+
exit 1
34+
fi
35+
36+
job_slug="`basename "$1" | sed 's/\.sh$//' | sed 's/[^a-zA-Z0-9]/-/g'`-$2"
37+
38+
sbatch <<EOT
39+
#!/bin/bash
40+
#SBATCH -Jshb-$job_slug # Job name
41+
#SBATCH --account=gts-sbryngelson3 # charge account
42+
#SBATCH -N1 # Number of nodes required
43+
$sbatch_device_opts
44+
#SBATCH -t 04:00:00 # Duration of the job (Ex: 15 mins)
45+
#SBATCH -q embers # QOS Name
46+
#SBATCH -o$job_slug.out # Combined output and error messages file
47+
#SBATCH -W # Do not exit until the submitted job terminates.
48+
49+
set -e
50+
set -x
51+
52+
cd "\$SLURM_SUBMIT_DIR"
53+
echo "Running in $(pwd):"
54+
55+
job_slug="$job_slug"
56+
job_device="$2"
57+
58+
. ./mfc.sh load -c p -m $2
59+
60+
$sbatch_script_contents
61+
62+
EOT

‎.github/workflows/phoenix/test.sh

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#!/bin/bash
2+
3+
build_opts=""
4+
if [ "$job_device" == "gpu" ]; then
5+
build_opts="--gpu"
6+
fi
7+
8+
./mfc.sh build -j $(nproc) $build_opts
9+
10+
n_test_threads=$(nproc)
11+
12+
if [ "$job_device" == "gpu" ]; then
13+
gpu_count=$(nvidia-smi -L | wc -l) # number of GPUs on node
14+
gpu_ids=$(seq -s ' ' 0 $(($gpu_count-1))) # 0,1,2,...,gpu_count-1
15+
device_opts="-g $gpu_ids"
16+
n_test_threads=`expr $gpu_count \* 2`
17+
fi
18+
19+
./mfc.sh test -a -b mpirun -j $n_test_threads $device_opts

‎.github/workflows/ci.yml renamed to ‎.github/workflows/test.yml

Lines changed: 7 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -114,19 +114,12 @@ jobs:
114114
- name: Clone
115115
uses: actions/checkout@v3
116116

117-
- name: Build
118-
run: |
119-
. ./mfc.sh load -c p -m gpu
120-
./mfc.sh build -j 2 $(if [ '${{ matrix.device }}' == 'gpu' ]; then echo '--gpu'; fi)
117+
- name: Build & Test
118+
run: bash .github/workflows/phoenix/submit.sh .github/workflows/phoenix/test.sh ${{ matrix.device }}
121119

122-
- name: Test
123-
run: |
124-
. ./mfc.sh load -c p -m gpu
125-
mv misc/run-phoenix-release-${{ matrix.device }}.sh ./
126-
sbatch run-phoenix-release-${{ matrix.device }}.sh
127-
128-
- name: Print
120+
- name: Archive Logs
121+
uses: actions/upload-artifact@v3
129122
if: always()
130-
run: |
131-
cat test.out
132-
123+
with:
124+
name: logs
125+
path: test-${{ matrix.device }}.out

‎README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
<img src="https://zenodo.org/badge/doi/10.1016/j.cpc.2020.107396.svg" />
1010
</a>
1111
<a href="https://github.com/MFlowCode/MFC/actions">
12-
<img src="https://github.com/MFlowCode/MFC/actions/workflows/ci.yml/badge.svg" />
12+
<img src="https://github.com/MFlowCode/MFC/actions/workflows/test.yml/badge.svg" />
1313
</a>
1414
<a href="https://lbesson.mit-license.org/">
1515
<img src="https://img.shields.io/badge/License-MIT-blue.svg" />

‎misc/run-phoenix-release-cpu.sh

Lines changed: 0 additions & 16 deletions
This file was deleted.

‎misc/run-phoenix-release-gpu.sh

Lines changed: 0 additions & 24 deletions
This file was deleted.

‎toolchain/bench.yaml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
- name: 1D_bubblescreen
2+
path: examples/1D_bubblescreen/case.py
3+
args: []

‎toolchain/mfc/args.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,9 @@ def add_common_arguments(p, mask = None):
122122
run.add_argument("--wait", action="store_true", default=False, help="(Batch) Wait for the job to finish.")
123123

124124
# === BENCH ===
125-
add_common_arguments(bench, "t")
125+
add_common_arguments(bench, "tjgn")
126+
bench.add_argument("output", metavar="OUTPUT", default=None, type=str, help="Path to the YAML output file to write the results to.")
127+
bench.add_argument(metavar="FORWARDED", default=[], dest='forwarded', nargs=argparse.REMAINDER, help="Arguments to forward to the ./mfc.sh run invocations.")
126128

127129
# === COUNT ===
128130
add_common_arguments(count, "g")

0 commit comments

Comments
 (0)