Skip to content

Commit 104a545

Browse files
henryleberrehenryleberre
committed
#49: GitHub CI Benchmarking
1 parent cf90cc0 commit 104a545

38 files changed

+531
-315
lines changed

‎.github/workflows/bench.yml

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
name: 'Benchmark'
2+
3+
on:
4+
pull_request:
5+
6+
jobs:
7+
self:
8+
name: Georgia Tech | Phoenix (NVHPC)
9+
if: github.repository == 'MFlowCode/MFC'
10+
strategy:
11+
matrix:
12+
device: ['cpu', 'gpu']
13+
runs-on:
14+
group: phoenix
15+
labels: self-hosted
16+
steps:
17+
- name: Clone - PR
18+
uses: actions/checkout@v3
19+
with:
20+
path: pr
21+
22+
- name: Clone - Master
23+
uses: actions/checkout@v3
24+
with:
25+
repository: henryleberre/MFC
26+
ref: master
27+
path: master
28+
29+
- name: Bench (Master v. PR)
30+
run: |
31+
(cd pr && bash .github/workflows/phoenix/submit.sh .github/workflows/phoenix/bench.sh ${{ matrix.device }}) &
32+
(cd master && bash .github/workflows/phoenix/submit.sh .github/workflows/phoenix/bench.sh ${{ matrix.device }}) &
33+
wait %1 && wait %2
34+
35+
- name: Generate Comment
36+
run: |
37+
COMMENT_MSG=`python3 .github/workflows/phoenix/compare.py master/bench-${{ matrix.device }}.yaml pr/bench-${{ matrix.device }}.yaml`
38+
echo "COMMENT_MSG=\"$COMMENT_MSG\"" >> $GITHUB_ENV
39+
40+
- name: Post Comment
41+
run: echo "$COMMENT_MSG"
42+
43+
- name: Archive Logs
44+
uses: actions/upload-artifact@v3
45+
if: always()
46+
with:
47+
name: logs-${{ matrix.device }}
48+
path: |
49+
${{ github.workspace }}/pr/bench-${{ matrix.device }}-*
50+
${{ github.workspace }}/master/bench-${{ matrix.device }}-*

‎.github/workflows/docker.yml

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,8 @@ jobs:
2222
password: ${{ secrets.DOCKER_PASSWORD }}
2323

2424
- name: Build & Publish thereto
25-
uses: docker/build-push-action@v4
25+
uses: docker/build-push-action@v3
2626
with:
2727
file: toolchain/Dockerfile
2828
push: true
2929
tags: ${{ secrets.DOCKER_USERNAME }}/mfc:latest
30-

‎.github/workflows/docs.yml

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,11 @@ jobs:
1111
docs:
1212
name: Build & Publish
1313
runs-on: ubuntu-latest
14-
14+
if: github.repository == 'MFlowCode/MFC'
15+
concurrency:
16+
group: docs-publish
17+
cancel-in-progress: true
18+
1519
steps:
1620
- uses: actions/checkout@v3
1721

@@ -46,4 +50,4 @@ jobs:
4650
git -C ../www push
4751
4852
# DOC_PUSH_URL should be of the format:
49-
# --> https://<username>:<token>@github.com/<username>/<repository>
53+
# --> https://<username>:<token>@github.com/<username>/<repository>

‎.github/workflows/lint.yml

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
name: Lint
2+
3+
on:
4+
push:
5+
6+
pull_request:
7+
8+
workflow_dispatch:
9+
10+
jobs:
11+
docs:
12+
name: Lint
13+
runs-on: ubuntu-latest
14+
15+
steps:
16+
- uses: actions/checkout@v3
17+
18+
- name: Lint
19+
run: ./mfc.sh lint

‎.github/workflows/phoenix/bench.sh

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
#!/bin/bash
2+
3+
n_ranks=$(nproc)
4+
5+
if [ "$job_device" == "gpu" ]; then
6+
n_ranks=$(nvidia-smi -L | wc -l) # number of GPUs on node
7+
gpu_ids=$(seq -s ' ' 0 $(($n_ranks-1))) # 0,1,2,...,gpu_count-1
8+
device_opts="--gpu -g $gpu_ids"
9+
fi
10+
11+
./mfc.sh bench -j $(nproc) -o "$job_slug.yaml" -- -b mpirun $device_opts -n $n_ranks

‎.github/workflows/phoenix/compare.py

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
#!/usr/bin/env python3
2+
3+
import argparse
4+
5+
import yaml
6+
7+
parser = argparse.ArgumentParser()
8+
parser.add_argument('master', metavar="MASTER", type=str)
9+
parser.add_argument('pr', metavar="PR", type=str)
10+
11+
args = parser.parse_args()
12+
13+
def load_cases(filepath):
14+
raw = yaml.safe_load(open(filepath))
15+
16+
cases = { case["slug"]: case for case in raw["cases"] }
17+
18+
for target in ["pre_process", "simulation"]:
19+
for case in cases:
20+
cases[case][target] = cases[case][target]["runtime"][target]
21+
22+
return cases
23+
24+
master, pr = load_cases(args.master), load_cases(args.pr)
25+
26+
master_keys = set(master.keys())
27+
pr_keys = set(pr.keys())
28+
29+
missing_cases = master_keys.symmetric_difference(pr_keys)
30+
31+
if len(missing_cases) > 0:
32+
print("**Warning:** The following cases are **missing** from master or this PR:\n")
33+
34+
for case in missing_cases:
35+
print(f" - {case}.")
36+
37+
print("")
38+
39+
speedups = {}
40+
41+
for case in master_keys.intersection(pr_keys):
42+
speedups[case] = {
43+
"pre_proess": pr[case]["pre_process"] / master[case]["pre_process"],
44+
"simulation": pr[case]["simulation"] / master[case]["simulation"],
45+
}
46+
47+
avg_speedup = sum([ speedups[case]["simulation"] for case in speedups ]) / len(speedups)
48+
49+
print(f"""\
50+
**[Benchmark Results]** Compared to Master, this PR's `simulation` is on average **~{avg_speedup:0.2f}x faster**.
51+
52+
| **Case** | **Master** | **PR** | **Speedup** |
53+
| -------- | ---------- | ------ | ----------- |\
54+
""")
55+
56+
for case in sorted(speedups.keys()):
57+
speedup = speedups[case]
58+
59+
print(f"| {case} | {master[case]['simulation']:0.2f}s | {pr[case]['simulation']:0.2f}s | {speedups[case]['simulation']:0.2f}x |")

‎.github/workflows/phoenix/submit.sh

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
#!/bin/bash
2+
3+
set -e
4+
5+
usage() {
6+
echo "Usage: $0 [script.sh] [cpu|gpu]"
7+
}
8+
9+
if [ ! -z "$1" ]; then
10+
sbatch_script_contents=`cat $1`
11+
else
12+
usage
13+
exit 1
14+
fi
15+
16+
sbatch_cpu_opts="\
17+
#SBATCH -p cpu-small # partition
18+
#SBATCH --ntasks-per-node=24 # Number of cores per node required
19+
#SBATCH --mem-per-cpu=2G # Memory per core\
20+
"
21+
22+
sbatch_gpu_opts="\
23+
#SBATCH -CV100-16GB
24+
#SBATCH -G2\
25+
"
26+
27+
if [ "$2" == "cpu" ]; then
28+
sbatch_device_opts="$sbatch_cpu_opts"
29+
elif [ "$2" == "gpu" ]; then
30+
sbatch_device_opts="$sbatch_gpu_opts"
31+
else
32+
usage
33+
exit 1
34+
fi
35+
36+
job_slug="`basename "$1" | sed 's/\.sh$//' | sed 's/[^a-zA-Z0-9]/-/g'`-$2"
37+
38+
sbatch <<EOT
39+
#!/bin/bash
40+
#SBATCH -Jshb-$job_slug # Job name
41+
#SBATCH --account=gts-sbryngelson3 # charge account
42+
#SBATCH -N1 # Number of nodes required
43+
$sbatch_device_opts
44+
#SBATCH -t 04:00:00 # Duration of the job (Ex: 15 mins)
45+
#SBATCH -q embers # QOS Name
46+
#SBATCH -o$job_slug.out # Combined output and error messages file
47+
#SBATCH -W # Do not exit until the submitted job terminates.
48+
49+
set -e
50+
set -x
51+
52+
cd "\$SLURM_SUBMIT_DIR"
53+
echo "Running in $(pwd):"
54+
55+
job_slug="$job_slug"
56+
job_device="$2"
57+
58+
. ./mfc.sh load -c p -m $2
59+
60+
$sbatch_script_contents
61+
62+
EOT

‎.github/workflows/phoenix/test.sh

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#!/bin/bash
2+
3+
build_opts=""
4+
if [ "$job_device" == "gpu" ]; then
5+
build_opts="--gpu"
6+
fi
7+
8+
./mfc.sh build -j $(nproc) $build_opts
9+
10+
n_test_threads=$(nproc)
11+
12+
if [ "$job_device" == "gpu" ]; then
13+
gpu_count=$(nvidia-smi -L | wc -l) # number of GPUs on node
14+
gpu_ids=$(seq -s ' ' 0 $(($gpu_count-1))) # 0,1,2,...,gpu_count-1
15+
device_opts="-g $gpu_ids"
16+
n_test_threads=`expr $gpu_count \* 2`
17+
fi
18+
19+
./mfc.sh test -a -j $n_test_threads $device_opts -- -b mpirun -f --bind-to none

‎.github/workflows/ci.yml renamed to ‎.github/workflows/test.yml

Lines changed: 7 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -114,19 +114,12 @@ jobs:
114114
- name: Clone
115115
uses: actions/checkout@v3
116116

117-
- name: Build
118-
run: |
119-
. ./mfc.sh load -c p -m gpu
120-
./mfc.sh build -j 2 $(if [ '${{ matrix.device }}' == 'gpu' ]; then echo '--gpu'; fi)
117+
- name: Build & Test
118+
run: bash .github/workflows/phoenix/submit.sh .github/workflows/phoenix/test.sh ${{ matrix.device }}
121119

122-
- name: Test
123-
run: |
124-
. ./mfc.sh load -c p -m gpu
125-
mv misc/run-phoenix-release-${{ matrix.device }}.sh ./
126-
sbatch run-phoenix-release-${{ matrix.device }}.sh
127-
128-
- name: Print
120+
- name: Archive Logs
121+
uses: actions/upload-artifact@v3
129122
if: always()
130-
run: |
131-
cat test.out
132-
123+
with:
124+
name: logs
125+
path: test-${{ matrix.device }}.out

‎README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
<img src="https://zenodo.org/badge/doi/10.1016/j.cpc.2020.107396.svg" />
1010
</a>
1111
<a href="https://github.com/MFlowCode/MFC/actions">
12-
<img src="https://github.com/MFlowCode/MFC/actions/workflows/ci.yml/badge.svg" />
12+
<img src="https://github.com/MFlowCode/MFC/actions/workflows/test.yml/badge.svg" />
1313
</a>
1414
<a href="https://lbesson.mit-license.org/">
1515
<img src="https://img.shields.io/badge/License-MIT-blue.svg" />

‎mfc.sh

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -439,10 +439,23 @@ if ! cmp "$(pwd)/toolchain/requirements.txt" "$(pwd)/build/requirements.txt" > /
439439
fi
440440

441441

442-
# Run the main.py bootstrap script
443-
python3 "$(pwd)/toolchain/mfc.py" "$@"
444-
code=$?
442+
if [ "$1" == "lint" ]; then
443+
shift
444+
445+
if ! command -v pylint > /dev/null 2>&1; then
446+
error "Couldn't find$MAGENTA pylint$COLOR_RESET. Please ensure it is discoverable."
447+
448+
exit 1
449+
fi
450+
451+
log "(venv) Running$MAGENTA pylint$COLOR_RESET on$MAGENTA MFC$COLOR_RESET."
445452

453+
pylint -d R1722,W0718,C0301,C0116,C0115,C0114,C0410,W0622,W0640 toolchain/
454+
else
455+
# Run the main.py bootstrap script
456+
python3 "$(pwd)/toolchain/mfc.py" "$@"
457+
code=$?
458+
fi
446459

447460
# Deactivate the Python virtualenv in case the user "source"'d this script
448461
log "(venv) Exiting the$MAGENTA Python$COLOR_RESET virtual environment."

‎misc/run-phoenix-release-cpu.sh

Lines changed: 0 additions & 16 deletions
This file was deleted.

‎misc/run-phoenix-release-gpu.sh

Lines changed: 0 additions & 24 deletions
This file was deleted.

‎toolchain/bench.yaml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
- slug: 3D_shockdroplet
2+
path: examples/3D_shockdroplet/case.py
3+
args: []
4+
- slug: 3D_sphbubcollapse
5+
path: examples/3D_sphbubcollapse/case.py
6+
args: []
7+
- slug: 3D_turb_mixing
8+
path: examples/3D_turb_mixing/case.py
9+
args: []

0 commit comments

Comments
 (0)