From e559f4c4ec3337a73ffd542b2024bd1b2e654bc9 Mon Sep 17 00:00:00 2001 From: Jan Dolinar Date: Tue, 26 Apr 2022 07:40:22 +0200 Subject: [PATCH] add memory measurement to benchmark script --- benchmark/benchmark.sh | 103 ++++++++++++++++++++++------------ benchmark/grammars/calc.peg | 5 +- benchmark/grammars/json.peg | 5 +- benchmark/grammars/kotlin.peg | 3 + 4 files changed, 79 insertions(+), 37 deletions(-) diff --git a/benchmark/benchmark.sh b/benchmark/benchmark.sh index 406b464..5fce3bb 100755 --- a/benchmark/benchmark.sh +++ b/benchmark/benchmark.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash # -# Generates, builds and runs parsers from grammar directory for each git reference supplied as argument. -# Each action is performed multiple times and the times are averaged. First reference is always -# taken as a "baseline" and others are compared to it. This should allow to compare how any given commit -# affects PackCCs performance. +# Generates, builds and runs parsers from grammars directory for each git reference supplied as argument. +# Each action is performed multiple times and the times are averaged. Peak memory consumption is also measured. +# First reference is always taken as a "baseline" and others are compared to it. This should allow to compare +# how any given commit affects PackCCs performance. # # Usage: # ./benchmark.sh ... @@ -35,61 +35,86 @@ format() { elif [ $((TIME / 1000)) -gt 10 ]; then echo "$((TIME / 1000)) us" else - echo "$((TIME)) ns" + echo "$TIME ns" + fi +} + +format_mem() { + MEM="$1" + if [ -z "$TIME_CMD" ]; then + echo "??? kB" + elif [ $((MEM / 1048576)) -gt 10 ]; then + echo "$((MEM / 1048576)) GB" + elif [ $((MEM / 1024)) -gt 10 ]; then + echo "$((MEM / 1024)) MB" + else + echo "$MEM kB" fi } measure() { COUNT="$1" shift + MEM=0 + if [ "$TIME_CMD" ]; then + MEM="$(${TIME_CMD[@]} -f %M "$@" 2>&1 >/dev/null)" + fi START="$(date '+%s%N')" for ((i=0; i /dev/null done END="$(date '+%s%N')" TIME=$(( END - START )) } -run() { - "$1" < "$2" > /dev/null -} - benchmark() { KEY="${GRAMMAR}_${REF//\//_}" NAME="tmp/parser_$KEY" echo "Generating $GRAMMAR parser in $REF ($GEN_REPEATS times)..." measure "$GEN_REPEATS" "$PACKCC" -o "$NAME" "$GRAMMAR_FILE" - GEN["$KEY"]=$TIME - echo " Repeated $GEN_REPEATS times in $(format $TIME)" + GEN_TIME["$KEY"]=$TIME + GEN_MEM["$KEY"]=$MEM + echo " Repeated $GEN_REPEATS times in $(format $TIME), peak memory $(format_mem $MEM)" echo "Building $GRAMMAR parser in $REF ($BUILD_REPEATS times)..." measure "$BUILD_REPEATS" $CC -I. "$NAME".c -o "$NAME" - BUILD["$KEY"]=$TIME - echo " Built $BUILD_REPEATS times in $(format $TIME)" + BUILD_TIME["$KEY"]=$TIME + BUILD_MEM["$KEY"]=$MEM + echo " Built $BUILD_REPEATS times in $(format $TIME), peak memory $(format_mem $MEM)" echo "Running $GRAMMAR parser in $REF ($RUN_REPEATS times)..." - measure "$RUN_REPEATS" run "./$NAME" "$INPUT" - RUN["$KEY"]=$TIME - echo " Repeated $RUN_REPEATS times in $(format $TIME)" + measure "$RUN_REPEATS" "./$NAME" "$INPUT" + RUN_TIME["$KEY"]=$TIME + RUN_MEM["$KEY"]=$MEM + echo " Repeated $RUN_REPEATS times in $(format $TIME), peak memory $(format_mem $MEM)" } print_table() { - declare -n RESULTS="$1" + declare -n RESULTS_TIME="${1}_TIME" + declare -n RESULTS_MEM="${1}_MEM" printf "%-12s" "" for REF in "${REFS[@]}"; do - printf "%-16s" "$REF" + printf "%-32s" "$REF" done printf "\n" + MEMORY=0 + RELATIVE_MEM="???" + COLOR_MEM=0 for GRAMMAR in "${GRAMMARS[@]}"; do printf "%-12s" "$GRAMMAR" for REF in "${REFS[@]}"; do KEY="${GRAMMAR}_${REF//\//_}" BASE="${GRAMMAR}_${REFS[0]//\//_}" - TIME="$((${RESULTS["$KEY"]} / RUN_REPEATS))" - RELATIVE="$((100 * RESULTS["$KEY"] / RESULTS["$BASE"]))" - COLOR=$((RELATIVE == 100 ? 0 : ( RELATIVE > 100 ? 31 : 32))) - printf "\033[0;${COLOR}m%-16s\033[0m" "$(format $TIME) ($RELATIVE%)" + TIME="$((${RESULTS_TIME["$KEY"]} / RUN_REPEATS))" + RELATIVE_TIME="$((100 * RESULTS_TIME["$KEY"] / RESULTS_TIME["$BASE"]))" + COLOR=$((RELATIVE_TIME == 100 ? 0 : ( RELATIVE_TIME > 100 ? 31 : 32))) + if [ "$TIME_CMD" ]; then + MEMORY="${RESULTS_MEM["$KEY"]}" + RELATIVE_MEM="$((100 * RESULTS_MEM["$KEY"] / RESULTS_MEM["$BASE"]))" + COLOR_MEM=$((RELATIVE_MEM == 100 ? 0 : ( RELATIVE_MEM > 100 ? 31 : 32))) + fi + printf "\033[0;${COLOR}m%-16s\033[0;${COLOR_MEM}m%-16s\033[0m" "$(format $TIME) ($RELATIVE_TIME%)" "$(format_mem $MEMORY) ($RELATIVE_MEM%)" done printf "\n" done @@ -97,17 +122,18 @@ print_table() { print_results() { echo - echo "Generation times:" - echo "=================" + echo "Generation performance:" + echo "=======================" print_table GEN echo - echo "Build times:" - echo "============" + echo "Build performance:" + echo "==================" print_table BUILD echo - echo "Run times:" - echo "==========" + echo "Run performance:" + echo "================" print_table RUN + echo } main() { @@ -116,13 +142,11 @@ main() { BENCHDIR="$(cd "$(dirname "$0")" && pwd)" ROOTDIR="$BENCHDIR/.." declare -a GRAMMARS=() - declare -A BUILD=() - declare -A GEN=() - declare -A RUN=() + declare -A BUILD_TIME GEN_TIME RUN_TIME BUILD_MEM GEN_MEM RUN_MEM - declare -i GEN_REPEATS="${GEN_REPEATS:-10}" - declare -i BUILD_REPEATS="${BUILD_REPEATS:-5}" - declare -i RUN_REPEATS="${RUN_REPEATS:-20}" + declare -i GEN_REPEATS="${GEN_REPEATS:-1}" + declare -i BUILD_REPEATS="${BUILD_REPEATS:-1}" + declare -i RUN_REPEATS="${RUN_REPEATS:-1}" CC="${CC:-cc -O2}" REFS=("$@") @@ -131,6 +155,15 @@ main() { exit 0 fi + if which busybox &> /dev/null; then + TIME_CMD=(busybox time) + elif which time &> /dev/null; then + TIME_CMD=("$(which time)") + else + echo "NOTE: No time command found, please install GNU time or busybox to measure memory consumption." + TIME_CMD="" + fi + START_REF="$(git name-rev --name-only HEAD)" trap "echo 'Returning to $START_REF...' && git checkout $START_REF" EXIT ERR INT diff --git a/benchmark/grammars/calc.peg b/benchmark/grammars/calc.peg index c7892d2..c86faeb 100644 --- a/benchmark/grammars/calc.peg +++ b/benchmark/grammars/calc.peg @@ -29,7 +29,10 @@ _ <- [ \t]* EOL <- '\n' / '\r\n' / '\r' / ';' %% -int main() { +int main(int argc, char **argv) { + if (argc > 1) { + freopen(argv[1], "r", stdin); + } calc_context_t *ctx = calc_create(NULL); while (calc_parse(ctx, NULL)); calc_destroy(ctx); diff --git a/benchmark/grammars/json.peg b/benchmark/grammars/json.peg index 9612224..51e95c0 100644 --- a/benchmark/grammars/json.peg +++ b/benchmark/grammars/json.peg @@ -12,7 +12,10 @@ null <- 'null' _ <- [ \n\r\t]* %% -int main() { +int main(int argc, char **argv) { + if (argc > 1) { + freopen(argv[1], "r", stdin); + } json_context_t *ctx = json_create(NULL); while (json_parse(ctx, NULL)); json_destroy(ctx); diff --git a/benchmark/grammars/kotlin.peg b/benchmark/grammars/kotlin.peg index bd47553..8207103 100644 --- a/benchmark/grammars/kotlin.peg +++ b/benchmark/grammars/kotlin.peg @@ -441,6 +441,9 @@ EOF <- !. %% int main(int argc, char **argv) { + if (argc > 1) { + freopen(argv[1], "r", stdin); + } int ret; pcc_context_t *ctx = pcc_create(NULL); while (pcc_parse(ctx, &ret));