From 18a7af45f1886706f74e78f79d54931851d1efbb Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 12 Nov 2024 11:49:07 +0100 Subject: [PATCH 01/11] Run benchmarks in CI --- buildkite/scripts/bench/run.sh | 33 +++++++++++++++++++ .../bench/snark_transaction_profiler.sh | 12 ------- buildkite/scripts/bench/zkapp_metrics.sh | 9 ----- buildkite/src/Command/Bench/Base.dhall | 5 ++- .../src/Jobs/Bench/HeapUsageStable.dhall | 23 +++++++++++++ .../src/Jobs/Bench/HeapUsageUnstable.dhall | 22 +++++++++++++ .../src/Jobs/Bench/LedgerExportStable.dhall | 23 +++++++++++++ .../src/Jobs/Bench/LedgerExportUnstable.dhall | 25 ++++++++++++++ buildkite/src/Jobs/Bench/MinaBaseStable.dhall | 23 +++++++++++++ .../src/Jobs/Bench/MinaBaseUnstable.dhall | 25 ++++++++++++++ .../src/Jobs/Bench/SnarkProfilerStable.dhall | 23 +++++++++++++ .../Jobs/Bench/SnarkProfilerUnstable.dhall | 25 ++++++++++++++ .../src/Jobs/Bench/ZkappLimitsStable.dhall | 23 +++++++++++++ .../src/Jobs/Bench/ZkappLimitsUnstable.dhall | 25 ++++++++++++++ scripts/debian/builder-helpers.sh | 3 ++ src/app/zkapp_limits/zkapp_limits.ml | 2 +- 16 files changed, 278 insertions(+), 23 deletions(-) create mode 100755 buildkite/scripts/bench/run.sh delete mode 100755 buildkite/scripts/bench/snark_transaction_profiler.sh delete mode 100755 buildkite/scripts/bench/zkapp_metrics.sh create mode 100755 buildkite/src/Jobs/Bench/HeapUsageStable.dhall create mode 100755 buildkite/src/Jobs/Bench/HeapUsageUnstable.dhall create mode 100755 buildkite/src/Jobs/Bench/LedgerExportStable.dhall create mode 100755 buildkite/src/Jobs/Bench/LedgerExportUnstable.dhall create mode 100755 buildkite/src/Jobs/Bench/MinaBaseStable.dhall create mode 100755 buildkite/src/Jobs/Bench/MinaBaseUnstable.dhall create mode 100755 buildkite/src/Jobs/Bench/SnarkProfilerStable.dhall create mode 100755 buildkite/src/Jobs/Bench/SnarkProfilerUnstable.dhall create mode 100755 buildkite/src/Jobs/Bench/ZkappLimitsStable.dhall create mode 100755 buildkite/src/Jobs/Bench/ZkappLimitsUnstable.dhall diff --git a/buildkite/scripts/bench/run.sh b/buildkite/scripts/bench/run.sh new file mode 100755 index 00000000000..ddc6bf1e5a2 --- /dev/null +++ b/buildkite/scripts/bench/run.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +set -eox pipefail + +# Don't prompt for answers during apt-get install + +YELLOW_THRESHOLD="0.1" +RED_THRESHOLD="0.3" + +source buildkite/scripts/bench/install.sh + +MAINLINE_BRANCHES="-m develop -m compatile -m master -m dkijania/build_performance_tooling_in_ci" +while [[ "$#" -gt 0 ]]; do case $1 in + heap-usage) BENCHMARK="heap-usage"; ;; + mina-base) BENCHMARK="mina-base"; ;; + ledger-export) + BENCHMARK="ledger-export" + EXTRA_ARGS="--genesis-ledger-path ./genesis_ledgers/devnet.json" + ;; + snark) + BENCHMARK="snark"; + K=1 + MAX_NUM_UPDATES=4 + MIN_NUM_UPDATES=2 + EXTRA_ARGS="--k ${K} --max-num-updates ${MAX_NUM_UPDATES} --min-num-updates ${MIN_NUM_UPDATES}" + ;; + zkapp) BENCHMARK="zkapp"; ;; + --yellow-threshold) YELLOW_THRESHOLD="$2"; shift;; + --red-threshold) RED_THRESHOLD="$2"; shift;; + *) echo "Unknown parameter passed: $1"; exit 1;; +esac; shift; done + +python3 ./scripts/benchmarks test --benchmark ${BENCHMARK} --branch ${BUILDKITE_BRANCH} --tmpfile ${BENCHMARK}.csv --yellow-threshold $YELLOW_THRESHOLD --red-threshold $RED_THRESHOLD $MAINLINE_BRANCHES $EXTRA_ARGS \ No newline at end of file diff --git a/buildkite/scripts/bench/snark_transaction_profiler.sh b/buildkite/scripts/bench/snark_transaction_profiler.sh deleted file mode 100755 index 2436efb92d4..00000000000 --- a/buildkite/scripts/bench/snark_transaction_profiler.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -set -eo pipefail - -source buildkite/scripts/bench/install.sh - -K=1 -MAX_NUM_UPDATES=4 -MIN_NUM_UPDATES=2 - -echo "-- Run Snark Transaction Profiler with parameters: --k ${K} --max-num-updates ${MAX_NUM_UPDATES} --min-num-updates ${MIN_NUM_UPDATES}" -python3 ./scripts/benchmarks run --benchmark snark --k ${K} --max-num-updates ${MAX_NUM_UPDATES} --min-num-updates ${MIN_NUM_UPDATES} --outfile snark.out diff --git a/buildkite/scripts/bench/zkapp_metrics.sh b/buildkite/scripts/bench/zkapp_metrics.sh deleted file mode 100755 index 829ece87cb1..00000000000 --- a/buildkite/scripts/bench/zkapp_metrics.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -set -eo pipefail - -source buildkite/scripts/bench/install.sh - -python3 ./scripts/benchmarks run --benchmark zkapp --outfile zakpp-out - -python3 ./scripts/benchmarks run --benchmark heap-usage --outfile heap-usage.out \ No newline at end of file diff --git a/buildkite/src/Command/Bench/Base.dhall b/buildkite/src/Command/Bench/Base.dhall index 3d2431943b6..0793bdd81a0 100644 --- a/buildkite/src/Command/Bench/Base.dhall +++ b/buildkite/src/Command/Bench/Base.dhall @@ -57,7 +57,7 @@ let command , commands = RunInToolchain.runInToolchain (Benchmarks.toEnvList Benchmarks.Type::{=}) - "./buildkite/scripts/benchmarks.sh ${spec.bench} --red-threshold ${Double/show + "./buildkite/scripts/bench/run.sh ${spec.bench} --red-threshold ${Double/show spec.redThreshold} --yellow-threshold ${Double/show spec.yellowThreshold}" , label = "Perf: ${spec.label}" @@ -79,6 +79,9 @@ let pipeline "dhall" , SelectFiles.contains "scripts/benchmark" , SelectFiles.contains "buildkite/scripts/benchmark" + , SelectFiles.exactly + "buildkite/src/Jobs/Bench/${spec.name}" + "dhall" ] # spec.additionalDirtyWhen , path = spec.path diff --git a/buildkite/src/Jobs/Bench/HeapUsageStable.dhall b/buildkite/src/Jobs/Bench/HeapUsageStable.dhall new file mode 100755 index 00000000000..aa51f683137 --- /dev/null +++ b/buildkite/src/Jobs/Bench/HeapUsageStable.dhall @@ -0,0 +1,23 @@ +let BenchBase = ../../Command/Bench/Base.dhall + +let Pipeline = ../../Pipeline/Dsl.dhall + +let S = ../../Lib/SelectFiles.dhall + +let PipelineMode = ../../Pipeline/Mode.dhall + +let name = "HeapUsageStable" + +let bench = "heap-usage" + +in Pipeline.build + ( BenchBase.pipeline + BenchBase.Spec::{ + , mode = PipelineMode.Type.Stable + , path = "Bench" + , name = name + , label = "Heap Usage Stable" + , key = bench + , bench = bench + } + ) diff --git a/buildkite/src/Jobs/Bench/HeapUsageUnstable.dhall b/buildkite/src/Jobs/Bench/HeapUsageUnstable.dhall new file mode 100755 index 00000000000..1ac1dd08994 --- /dev/null +++ b/buildkite/src/Jobs/Bench/HeapUsageUnstable.dhall @@ -0,0 +1,22 @@ +let BenchBase = ../../Command/Bench/Base.dhall + +let Pipeline = ../../Pipeline/Dsl.dhall + +let S = ../../Lib/SelectFiles.dhall + +let name = "HeapUsageUnstable" + +let bench = "heap-usage" + +in Pipeline.build + ( BenchBase.pipeline + BenchBase.Spec::{ + , path = "Bench" + , name = name + , label = "Heap Usage Pr" + , key = bench + , bench = bench + , yellowThreshold = 0.1 + , redThreshold = 0.3 + } + ) diff --git a/buildkite/src/Jobs/Bench/LedgerExportStable.dhall b/buildkite/src/Jobs/Bench/LedgerExportStable.dhall new file mode 100755 index 00000000000..e8a53c48318 --- /dev/null +++ b/buildkite/src/Jobs/Bench/LedgerExportStable.dhall @@ -0,0 +1,23 @@ +let BenchBase = ../../Command/Bench/Base.dhall + +let Pipeline = ../../Pipeline/Dsl.dhall + +let S = ../../Lib/SelectFiles.dhall + +let PipelineMode = ../../Pipeline/Mode.dhall + +let name = "LedgerExportStable" + +let bench = "ledger-export" + +in Pipeline.build + ( BenchBase.pipeline + BenchBase.Spec::{ + , mode = PipelineMode.Type.Stable + , path = "Bench" + , name = name + , label = "Ledger Export Stable" + , key = bench + , bench = bench + } + ) diff --git a/buildkite/src/Jobs/Bench/LedgerExportUnstable.dhall b/buildkite/src/Jobs/Bench/LedgerExportUnstable.dhall new file mode 100755 index 00000000000..7bea6ec651b --- /dev/null +++ b/buildkite/src/Jobs/Bench/LedgerExportUnstable.dhall @@ -0,0 +1,25 @@ +let BenchBase = ../../Command/Bench/Base.dhall + +let Pipeline = ../../Pipeline/Dsl.dhall + +let Size = ../../Command/Size.dhall + +let S = ../../Lib/SelectFiles.dhall + +let name = "LedgerExportUnstable" + +let bench = "ledger-export" + +in Pipeline.build + ( BenchBase.pipeline + BenchBase.Spec::{ + , path = "Bench" + , name = name + , label = "Ledger Export Pr" + , size = Size.Small + , bench = bench + , key = bench + , yellowThreshold = 0.1 + , redThreshold = 0.3 + } + ) diff --git a/buildkite/src/Jobs/Bench/MinaBaseStable.dhall b/buildkite/src/Jobs/Bench/MinaBaseStable.dhall new file mode 100755 index 00000000000..d870a13e007 --- /dev/null +++ b/buildkite/src/Jobs/Bench/MinaBaseStable.dhall @@ -0,0 +1,23 @@ +let BenchBase = ../../Command/Bench/Base.dhall + +let Pipeline = ../../Pipeline/Dsl.dhall + +let PipelineMode = ../../Pipeline/Mode.dhall + +let S = ../../Lib/SelectFiles.dhall + +let name = "MinaBaseStable" + +let bench = "mina-base" + +in Pipeline.build + ( BenchBase.pipeline + BenchBase.Spec::{ + , mode = PipelineMode.Type.Stable + , path = "Bench" + , name = name + , label = "Mina Base Stable" + , key = bench + , bench = bench + } + ) diff --git a/buildkite/src/Jobs/Bench/MinaBaseUnstable.dhall b/buildkite/src/Jobs/Bench/MinaBaseUnstable.dhall new file mode 100755 index 00000000000..b78b707a9b1 --- /dev/null +++ b/buildkite/src/Jobs/Bench/MinaBaseUnstable.dhall @@ -0,0 +1,25 @@ +let BenchBase = ../../Command/Bench/Base.dhall + +let Pipeline = ../../Pipeline/Dsl.dhall + +let Size = ../../Command/Size.dhall + +let S = ../../Lib/SelectFiles.dhall + +let name = "MinaBaseUnstable" + +let bench = "mina-base" + +in Pipeline.build + ( BenchBase.pipeline + BenchBase.Spec::{ + , path = "Bench" + , name = name + , label = "Mina Base Pr" + , size = Size.Small + , bench = bench + , key = bench + , yellowThreshold = 0.1 + , redThreshold = 0.3 + } + ) diff --git a/buildkite/src/Jobs/Bench/SnarkProfilerStable.dhall b/buildkite/src/Jobs/Bench/SnarkProfilerStable.dhall new file mode 100755 index 00000000000..803f2d8cff6 --- /dev/null +++ b/buildkite/src/Jobs/Bench/SnarkProfilerStable.dhall @@ -0,0 +1,23 @@ +let BenchBase = ../../Command/Bench/Base.dhall + +let Pipeline = ../../Pipeline/Dsl.dhall + +let S = ../../Lib/SelectFiles.dhall + +let PipelineMode = ../../Pipeline/Mode.dhall + +let name = "SnarkProfilerStable" + +let bench = "snark" + +in Pipeline.build + ( BenchBase.pipeline + BenchBase.Spec::{ + , mode = PipelineMode.Type.Stable + , path = "Bench" + , name = name + , label = "Snark Profiler Stable" + , key = bench + , bench = bench + } + ) diff --git a/buildkite/src/Jobs/Bench/SnarkProfilerUnstable.dhall b/buildkite/src/Jobs/Bench/SnarkProfilerUnstable.dhall new file mode 100755 index 00000000000..de6880e2893 --- /dev/null +++ b/buildkite/src/Jobs/Bench/SnarkProfilerUnstable.dhall @@ -0,0 +1,25 @@ +let BenchBase = ../../Command/Bench/Base.dhall + +let Pipeline = ../../Pipeline/Dsl.dhall + +let Size = ../../Command/Size.dhall + +let S = ../../Lib/SelectFiles.dhall + +let name = "SnarkProfilerUnstable" + +let bench = "snark" + +in Pipeline.build + ( BenchBase.pipeline + BenchBase.Spec::{ + , path = "Bench" + , name = name + , label = "Snark Profiler Pr" + , size = Size.Small + , bench = bench + , key = bench + , yellowThreshold = 0.1 + , redThreshold = 0.3 + } + ) diff --git a/buildkite/src/Jobs/Bench/ZkappLimitsStable.dhall b/buildkite/src/Jobs/Bench/ZkappLimitsStable.dhall new file mode 100755 index 00000000000..bcdafd8a76c --- /dev/null +++ b/buildkite/src/Jobs/Bench/ZkappLimitsStable.dhall @@ -0,0 +1,23 @@ +let BenchBase = ../../Command/Bench/Base.dhall + +let Pipeline = ../../Pipeline/Dsl.dhall + +let S = ../../Lib/SelectFiles.dhall + +let PipelineMode = ../../Pipeline/Mode.dhall + +let name = "ZkappLimitsStable" + +let bench = "zkapp" + +in Pipeline.build + ( BenchBase.pipeline + BenchBase.Spec::{ + , mode = PipelineMode.Type.Stable + , path = "Bench" + , name = name + , label = "Zkapp Limits Stable" + , key = bench + , bench = bench + } + ) diff --git a/buildkite/src/Jobs/Bench/ZkappLimitsUnstable.dhall b/buildkite/src/Jobs/Bench/ZkappLimitsUnstable.dhall new file mode 100755 index 00000000000..0603a97dea3 --- /dev/null +++ b/buildkite/src/Jobs/Bench/ZkappLimitsUnstable.dhall @@ -0,0 +1,25 @@ +let BenchBase = ../../Command/Bench/Base.dhall + +let Pipeline = ../../Pipeline/Dsl.dhall + +let Size = ../../Command/Size.dhall + +let S = ../../Lib/SelectFiles.dhall + +let name = "ZkappLimitsUnstable" + +let bench = "zkapp" + +in Pipeline.build + ( BenchBase.pipeline + BenchBase.Spec::{ + , path = "Bench" + , name = name + , label = "Zkapp Limits Pr" + , size = Size.Small + , key = bench + , bench = bench + , yellowThreshold = 0.1 + , redThreshold = 0.3 + } + ) diff --git a/scripts/debian/builder-helpers.sh b/scripts/debian/builder-helpers.sh index af9a92b1c1e..7a8a2c7db60 100755 --- a/scripts/debian/builder-helpers.sh +++ b/scripts/debian/builder-helpers.sh @@ -241,6 +241,9 @@ build_functional_test_suite_deb() { # Binaries cp ./default/src/test/command_line_tests/command_line_tests.exe "${BUILDDIR}/usr/local/bin/mina-command-line-tests" + cp ./default/src/app/benchmarks/benchmarks.exe "${BUILDDIR}/usr/local/bin/mina-benchmarks" + cp ./default/src/app/ledger_export_bench/ledger_export_benchmark.exe "${BUILDDIR}/usr/local/bin/mina-ledger-export-benchmark" + cp ./default/src/app/disk_caching_stats/disk_caching_stats.exe "${BUILDDIR}/usr/local/bin/mina-disk-caching-stats" cp ./default/src/app/heap_usage/heap_usage.exe "${BUILDDIR}/usr/local/bin/mina-heap-usage" cp ./default/src/app/zkapp_limits/zkapp_limits.exe "${BUILDDIR}/usr/local/bin/mina-zkapp-limits" cp ./default/src/test/archive/patch_archive_test/patch_archive_test.exe "${BUILDDIR}/usr/local/bin/mina-patch-archive-test" diff --git a/src/app/zkapp_limits/zkapp_limits.ml b/src/app/zkapp_limits/zkapp_limits.ml index 47f78d8f873..d2cf0345ba6 100644 --- a/src/app/zkapp_limits/zkapp_limits.ml +++ b/src/app/zkapp_limits/zkapp_limits.ml @@ -11,7 +11,7 @@ let main () = max_event_elements ; printf "max field elements for actions per transaction: %d\n" max_action_elements ; - printf "All possible zkApp account update combinations" ; + printf "All possible zkApp account update combinations: \n" ; List.iter values ~f:(fun proofs -> List.iter values ~f:(fun signed_single_segments -> List.iter values ~f:(fun signed_pair_segments -> From ea868e2f3a848c5deb27a3abeeff82ebbfeaa3e3 Mon Sep 17 00:00:00 2001 From: dkijania Date: Thu, 14 Nov 2024 18:32:22 +0100 Subject: [PATCH 02/11] clean up benches --- buildkite/src/Command/Bench/Base.dhall | 13 ++-- buildkite/src/Constants/DebianVersions.dhall | 2 - .../src/Jobs/Bench/HeapUsageStable.dhall | 4 +- .../src/Jobs/Bench/HeapUsageUnstable.dhall | 4 +- .../src/Jobs/Bench/LedgerExportStable.dhall | 4 +- .../src/Jobs/Bench/LedgerExportUnstable.dhall | 4 +- buildkite/src/Jobs/Bench/MinaBaseStable.dhall | 4 +- .../src/Jobs/Bench/MinaBaseUnstable.dhall | 4 +- .../src/Jobs/Bench/SnarkProfilerStable.dhall | 4 +- .../Jobs/Bench/SnarkProfilerUnstable.dhall | 4 +- .../src/Jobs/Bench/ZkappLimitsStable.dhall | 4 +- .../src/Jobs/Bench/ZkappLimitsUnstable.dhall | 4 +- .../src/Jobs/Test/RunSnarkProfiler.dhall | 67 ------------------- buildkite/src/Jobs/Test/ZkappMetrics.dhall | 57 ---------------- 14 files changed, 17 insertions(+), 162 deletions(-) delete mode 100644 buildkite/src/Jobs/Test/RunSnarkProfiler.dhall delete mode 100644 buildkite/src/Jobs/Test/ZkappMetrics.dhall diff --git a/buildkite/src/Command/Bench/Base.dhall b/buildkite/src/Command/Bench/Base.dhall index 0793bdd81a0..be3016540cd 100644 --- a/buildkite/src/Command/Bench/Base.dhall +++ b/buildkite/src/Command/Bench/Base.dhall @@ -58,9 +58,9 @@ let command RunInToolchain.runInToolchain (Benchmarks.toEnvList Benchmarks.Type::{=}) "./buildkite/scripts/bench/run.sh ${spec.bench} --red-threshold ${Double/show - spec.redThreshold} --yellow-threshold ${Double/show - spec.yellowThreshold}" - , label = "Perf: ${spec.label}" + spec.redThreshold} --yellow-threshold ${Double/show + spec.yellowThreshold}" + , label = "Perf: ${spec.name} ${PipelineMode.capitalName spec.mode}" , key = spec.key , target = spec.size , docker = None Docker.Type @@ -77,10 +77,11 @@ let pipeline , SelectFiles.exactly "buildkite/src/Command/Bench/Base" "dhall" + , SelectFiles.exactly "buildkite/scripts/bench/install" "sh" + , SelectFiles.exactly "buildkite/scripts/bench/run" "sh" , SelectFiles.contains "scripts/benchmark" - , SelectFiles.contains "buildkite/scripts/benchmark" - , SelectFiles.exactly - "buildkite/src/Jobs/Bench/${spec.name}" + , SelectFiles.exactly + "buildkite/src/Jobs/Bench/${spec.name}" "dhall" ] # spec.additionalDirtyWhen diff --git a/buildkite/src/Constants/DebianVersions.dhall b/buildkite/src/Constants/DebianVersions.dhall index 138a0533803..5b659225ff1 100644 --- a/buildkite/src/Constants/DebianVersions.dhall +++ b/buildkite/src/Constants/DebianVersions.dhall @@ -101,8 +101,6 @@ let minimalDirtyWhen = , S.exactly "buildkite/scripts/build-artifact" "sh" , S.exactly "buildkite/scripts/build-hardfork-package" "sh" , S.exactly "buildkite/scripts/check-compatibility" "sh" - , S.exactly "buildkite/src/Jobs/Test/RunSnarkProfiler" "dhall" - , S.exactly "buildkite/scripts/bench/snark_transaction_profiler" "sh" , S.exactly "buildkite/scripts/version-linter" "sh" , S.exactly "scripts/version-linter" "py" ] diff --git a/buildkite/src/Jobs/Bench/HeapUsageStable.dhall b/buildkite/src/Jobs/Bench/HeapUsageStable.dhall index aa51f683137..51e97d52684 100755 --- a/buildkite/src/Jobs/Bench/HeapUsageStable.dhall +++ b/buildkite/src/Jobs/Bench/HeapUsageStable.dhall @@ -2,8 +2,6 @@ let BenchBase = ../../Command/Bench/Base.dhall let Pipeline = ../../Pipeline/Dsl.dhall -let S = ../../Lib/SelectFiles.dhall - let PipelineMode = ../../Pipeline/Mode.dhall let name = "HeapUsageStable" @@ -16,7 +14,7 @@ in Pipeline.build , mode = PipelineMode.Type.Stable , path = "Bench" , name = name - , label = "Heap Usage Stable" + , label = "Heap Usage" , key = bench , bench = bench } diff --git a/buildkite/src/Jobs/Bench/HeapUsageUnstable.dhall b/buildkite/src/Jobs/Bench/HeapUsageUnstable.dhall index 1ac1dd08994..192782effc5 100755 --- a/buildkite/src/Jobs/Bench/HeapUsageUnstable.dhall +++ b/buildkite/src/Jobs/Bench/HeapUsageUnstable.dhall @@ -2,8 +2,6 @@ let BenchBase = ../../Command/Bench/Base.dhall let Pipeline = ../../Pipeline/Dsl.dhall -let S = ../../Lib/SelectFiles.dhall - let name = "HeapUsageUnstable" let bench = "heap-usage" @@ -13,7 +11,7 @@ in Pipeline.build BenchBase.Spec::{ , path = "Bench" , name = name - , label = "Heap Usage Pr" + , label = "Heap Usage" , key = bench , bench = bench , yellowThreshold = 0.1 diff --git a/buildkite/src/Jobs/Bench/LedgerExportStable.dhall b/buildkite/src/Jobs/Bench/LedgerExportStable.dhall index e8a53c48318..429f1c7043c 100755 --- a/buildkite/src/Jobs/Bench/LedgerExportStable.dhall +++ b/buildkite/src/Jobs/Bench/LedgerExportStable.dhall @@ -2,8 +2,6 @@ let BenchBase = ../../Command/Bench/Base.dhall let Pipeline = ../../Pipeline/Dsl.dhall -let S = ../../Lib/SelectFiles.dhall - let PipelineMode = ../../Pipeline/Mode.dhall let name = "LedgerExportStable" @@ -16,7 +14,7 @@ in Pipeline.build , mode = PipelineMode.Type.Stable , path = "Bench" , name = name - , label = "Ledger Export Stable" + , label = "Ledger Export" , key = bench , bench = bench } diff --git a/buildkite/src/Jobs/Bench/LedgerExportUnstable.dhall b/buildkite/src/Jobs/Bench/LedgerExportUnstable.dhall index 7bea6ec651b..a24b6c75f7b 100755 --- a/buildkite/src/Jobs/Bench/LedgerExportUnstable.dhall +++ b/buildkite/src/Jobs/Bench/LedgerExportUnstable.dhall @@ -4,8 +4,6 @@ let Pipeline = ../../Pipeline/Dsl.dhall let Size = ../../Command/Size.dhall -let S = ../../Lib/SelectFiles.dhall - let name = "LedgerExportUnstable" let bench = "ledger-export" @@ -15,7 +13,7 @@ in Pipeline.build BenchBase.Spec::{ , path = "Bench" , name = name - , label = "Ledger Export Pr" + , label = "Ledger Export" , size = Size.Small , bench = bench , key = bench diff --git a/buildkite/src/Jobs/Bench/MinaBaseStable.dhall b/buildkite/src/Jobs/Bench/MinaBaseStable.dhall index d870a13e007..80eba77d051 100755 --- a/buildkite/src/Jobs/Bench/MinaBaseStable.dhall +++ b/buildkite/src/Jobs/Bench/MinaBaseStable.dhall @@ -4,8 +4,6 @@ let Pipeline = ../../Pipeline/Dsl.dhall let PipelineMode = ../../Pipeline/Mode.dhall -let S = ../../Lib/SelectFiles.dhall - let name = "MinaBaseStable" let bench = "mina-base" @@ -16,7 +14,7 @@ in Pipeline.build , mode = PipelineMode.Type.Stable , path = "Bench" , name = name - , label = "Mina Base Stable" + , label = "Mina Base" , key = bench , bench = bench } diff --git a/buildkite/src/Jobs/Bench/MinaBaseUnstable.dhall b/buildkite/src/Jobs/Bench/MinaBaseUnstable.dhall index b78b707a9b1..9e90b21267e 100755 --- a/buildkite/src/Jobs/Bench/MinaBaseUnstable.dhall +++ b/buildkite/src/Jobs/Bench/MinaBaseUnstable.dhall @@ -4,8 +4,6 @@ let Pipeline = ../../Pipeline/Dsl.dhall let Size = ../../Command/Size.dhall -let S = ../../Lib/SelectFiles.dhall - let name = "MinaBaseUnstable" let bench = "mina-base" @@ -15,7 +13,7 @@ in Pipeline.build BenchBase.Spec::{ , path = "Bench" , name = name - , label = "Mina Base Pr" + , label = "Mina Base" , size = Size.Small , bench = bench , key = bench diff --git a/buildkite/src/Jobs/Bench/SnarkProfilerStable.dhall b/buildkite/src/Jobs/Bench/SnarkProfilerStable.dhall index 803f2d8cff6..07717a4db19 100755 --- a/buildkite/src/Jobs/Bench/SnarkProfilerStable.dhall +++ b/buildkite/src/Jobs/Bench/SnarkProfilerStable.dhall @@ -2,8 +2,6 @@ let BenchBase = ../../Command/Bench/Base.dhall let Pipeline = ../../Pipeline/Dsl.dhall -let S = ../../Lib/SelectFiles.dhall - let PipelineMode = ../../Pipeline/Mode.dhall let name = "SnarkProfilerStable" @@ -16,7 +14,7 @@ in Pipeline.build , mode = PipelineMode.Type.Stable , path = "Bench" , name = name - , label = "Snark Profiler Stable" + , label = "Snark Profiler" , key = bench , bench = bench } diff --git a/buildkite/src/Jobs/Bench/SnarkProfilerUnstable.dhall b/buildkite/src/Jobs/Bench/SnarkProfilerUnstable.dhall index de6880e2893..c433de99160 100755 --- a/buildkite/src/Jobs/Bench/SnarkProfilerUnstable.dhall +++ b/buildkite/src/Jobs/Bench/SnarkProfilerUnstable.dhall @@ -4,8 +4,6 @@ let Pipeline = ../../Pipeline/Dsl.dhall let Size = ../../Command/Size.dhall -let S = ../../Lib/SelectFiles.dhall - let name = "SnarkProfilerUnstable" let bench = "snark" @@ -15,7 +13,7 @@ in Pipeline.build BenchBase.Spec::{ , path = "Bench" , name = name - , label = "Snark Profiler Pr" + , label = "Snark Profiler" , size = Size.Small , bench = bench , key = bench diff --git a/buildkite/src/Jobs/Bench/ZkappLimitsStable.dhall b/buildkite/src/Jobs/Bench/ZkappLimitsStable.dhall index bcdafd8a76c..9147044997f 100755 --- a/buildkite/src/Jobs/Bench/ZkappLimitsStable.dhall +++ b/buildkite/src/Jobs/Bench/ZkappLimitsStable.dhall @@ -2,8 +2,6 @@ let BenchBase = ../../Command/Bench/Base.dhall let Pipeline = ../../Pipeline/Dsl.dhall -let S = ../../Lib/SelectFiles.dhall - let PipelineMode = ../../Pipeline/Mode.dhall let name = "ZkappLimitsStable" @@ -16,7 +14,7 @@ in Pipeline.build , mode = PipelineMode.Type.Stable , path = "Bench" , name = name - , label = "Zkapp Limits Stable" + , label = "Zkapp Limits" , key = bench , bench = bench } diff --git a/buildkite/src/Jobs/Bench/ZkappLimitsUnstable.dhall b/buildkite/src/Jobs/Bench/ZkappLimitsUnstable.dhall index 0603a97dea3..b54af75eddc 100755 --- a/buildkite/src/Jobs/Bench/ZkappLimitsUnstable.dhall +++ b/buildkite/src/Jobs/Bench/ZkappLimitsUnstable.dhall @@ -4,8 +4,6 @@ let Pipeline = ../../Pipeline/Dsl.dhall let Size = ../../Command/Size.dhall -let S = ../../Lib/SelectFiles.dhall - let name = "ZkappLimitsUnstable" let bench = "zkapp" @@ -15,7 +13,7 @@ in Pipeline.build BenchBase.Spec::{ , path = "Bench" , name = name - , label = "Zkapp Limits Pr" + , label = "Zkapp Limits" , size = Size.Small , key = bench , bench = bench diff --git a/buildkite/src/Jobs/Test/RunSnarkProfiler.dhall b/buildkite/src/Jobs/Test/RunSnarkProfiler.dhall deleted file mode 100644 index 9bfa98eb8a5..00000000000 --- a/buildkite/src/Jobs/Test/RunSnarkProfiler.dhall +++ /dev/null @@ -1,67 +0,0 @@ -let S = ../../Lib/SelectFiles.dhall - -let Pipeline = ../../Pipeline/Dsl.dhall - -let PipelineTag = ../../Pipeline/Tag.dhall - -let JobSpec = ../../Pipeline/JobSpec.dhall - -let DebianVersions = ../../Constants/DebianVersions.dhall - -let Profiles = ../../Constants/Profiles.dhall - -let Command = ../../Command/Base.dhall - -let RunInToolchain = ../../Command/RunInToolchain.dhall - -let Docker = ../../Command/Docker/Type.dhall - -let Size = ../../Command/Size.dhall - -let dependsOn = - DebianVersions.dependsOn - DebianVersions.DebVersion.Bullseye - Profiles.Type.Standard - -let buildTestCmd - : Size -> List Command.TaggedKey.Type -> Command.Type - = \(cmd_target : Size) - -> \(dependsOn : List Command.TaggedKey.Type) - -> Command.build - Command.Config::{ - , commands = - RunInToolchain.runInToolchain - ([] : List Text) - "buildkite/scripts/bench/snark_transaction_profiler.sh" - , label = "Snark Transaction Profiler" - , key = "snark-transaction-profiler" - , target = cmd_target - , docker = None Docker.Type - , artifact_paths = [ S.contains "core_dumps/*" ] - , depends_on = dependsOn - } - -in Pipeline.build - Pipeline.Config::{ - , spec = - let lintDirtyWhen = - [ S.strictlyStart (S.contains "src") - , S.exactly "buildkite/src/Jobs/Test/RunSnarkProfiler" "dhall" - , S.exactly - "buildkite/scripts/bench/snark_transaction_profiler" - "sh" - , S.strictlyStart (S.contains "scripts/benchmarks") - ] - - in JobSpec::{ - , dirtyWhen = lintDirtyWhen - , path = "Test" - , name = "RunSnarkProfiler" - , tags = - [ PipelineTag.Type.Long - , PipelineTag.Type.Test - , PipelineTag.Type.Stable - ] - } - , steps = [ buildTestCmd Size.Small dependsOn ] - } diff --git a/buildkite/src/Jobs/Test/ZkappMetrics.dhall b/buildkite/src/Jobs/Test/ZkappMetrics.dhall deleted file mode 100644 index 90117a87a34..00000000000 --- a/buildkite/src/Jobs/Test/ZkappMetrics.dhall +++ /dev/null @@ -1,57 +0,0 @@ -let S = ../../Lib/SelectFiles.dhall - -let Pipeline = ../../Pipeline/Dsl.dhall - -let PipelineTag = ../../Pipeline/Tag.dhall - -let Command = ../../Command/Base.dhall - -let RunInToolchain = ../../Command/RunInToolchain.dhall - -let DebianVersions = ../../Constants/DebianVersions.dhall - -let Profiles = ../../Constants/Profiles.dhall - -let Docker = ../../Command/Docker/Type.dhall - -let Size = ../../Command/Size.dhall - -let JobSpec = ../../Pipeline/JobSpec.dhall - -let dependsOn = - DebianVersions.dependsOn - DebianVersions.DebVersion.Bullseye - Profiles.Type.Standard - -in Pipeline.build - Pipeline.Config::{ - , spec = JobSpec::{ - , dirtyWhen = - [ S.strictlyStart (S.contains "buildkite/src/Jobs/Test/ZkappMetrics") - , S.strictlyStart (S.contains "src") - , S.exactly "buildkite/scripts/bench/zkapp_metrics" "sh" - , S.strictlyStart (S.contains "scripts/benchmarks") - ] - , path = "Test" - , name = "ZkappMetrics" - , tags = - [ PipelineTag.Type.Fast - , PipelineTag.Type.Test - , PipelineTag.Type.Stable - ] - } - , steps = - [ Command.build - Command.Config::{ - , commands = - RunInToolchain.runInToolchain - ([] : List Text) - "./buildkite/scripts/bench/zkapp_metrics.sh" - , label = "Zkapp Metrics" - , key = "zkapp-metrics" - , target = Size.Medium - , docker = None Docker.Type - , depends_on = dependsOn - } - ] - } From 6d6051dbcd0a07e7ec2283a9079f41d24f229645 Mon Sep 17 00:00:00 2001 From: dkijania Date: Fri, 15 Nov 2024 16:39:57 +0100 Subject: [PATCH 03/11] fix name rendering and missing extra args --- buildkite/scripts/bench/run.sh | 1 + buildkite/src/Command/Bench/Base.dhall | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/buildkite/scripts/bench/run.sh b/buildkite/scripts/bench/run.sh index ddc6bf1e5a2..af0c7e797b1 100755 --- a/buildkite/scripts/bench/run.sh +++ b/buildkite/scripts/bench/run.sh @@ -6,6 +6,7 @@ set -eox pipefail YELLOW_THRESHOLD="0.1" RED_THRESHOLD="0.3" +EXTRA_ARGS="" source buildkite/scripts/bench/install.sh diff --git a/buildkite/src/Command/Bench/Base.dhall b/buildkite/src/Command/Bench/Base.dhall index be3016540cd..fd4167dd909 100644 --- a/buildkite/src/Command/Bench/Base.dhall +++ b/buildkite/src/Command/Bench/Base.dhall @@ -60,7 +60,7 @@ let command "./buildkite/scripts/bench/run.sh ${spec.bench} --red-threshold ${Double/show spec.redThreshold} --yellow-threshold ${Double/show spec.yellowThreshold}" - , label = "Perf: ${spec.name} ${PipelineMode.capitalName spec.mode}" + , label = "Perf: ${spec.label} ${PipelineMode.capitalName spec.mode}" , key = spec.key , target = spec.size , docker = None Docker.Type From 728bf0685e39caae2f64fc1a84253637b19a6804 Mon Sep 17 00:00:00 2001 From: dkijania Date: Sun, 17 Nov 2024 21:56:31 +0100 Subject: [PATCH 04/11] lint dhall --- buildkite/src/Command/Bench/Base.dhall | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/buildkite/src/Command/Bench/Base.dhall b/buildkite/src/Command/Bench/Base.dhall index fd4167dd909..84c5422d8f8 100644 --- a/buildkite/src/Command/Bench/Base.dhall +++ b/buildkite/src/Command/Bench/Base.dhall @@ -60,7 +60,8 @@ let command "./buildkite/scripts/bench/run.sh ${spec.bench} --red-threshold ${Double/show spec.redThreshold} --yellow-threshold ${Double/show spec.yellowThreshold}" - , label = "Perf: ${spec.label} ${PipelineMode.capitalName spec.mode}" + , label = + "Perf: ${spec.label} ${PipelineMode.capitalName spec.mode}" , key = spec.key , target = spec.size , docker = None Docker.Type From d12b75c20baf9574d950e7d65596cd26d5e2e1da Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 19 Nov 2024 20:56:56 +0100 Subject: [PATCH 05/11] fix env vars naming --- buildkite/src/Constants/Benchmarks.dhall | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/buildkite/src/Constants/Benchmarks.dhall b/buildkite/src/Constants/Benchmarks.dhall index d303dd42499..7747600e4d0 100644 --- a/buildkite/src/Constants/Benchmarks.dhall +++ b/buildkite/src/Constants/Benchmarks.dhall @@ -1,19 +1,19 @@ let Spec = { Type = { tokenEnvName : Text, bucket : Text, org : Text, host : Text } , default = - { tokenEnvName = "\\\${INFLUX_TOKEN}" - , bucket = "\\\${INFLUX_BUCKET_NAME}" - , org = "\\\${INFLUX_ORG}" - , host = "\\\${INFLUX_HOST}" + { tokenEnvName = "\\\${INFLUXDB_TOKEN}" + , bucket = "\\\${INFLUXDB_BUCKET_NAME}" + , org = "\\\${INFLUXDB_ORG}" + , host = "\\\${INFLUXDB_HOST}" } } let toEnvList = \(spec : Spec.Type) - -> [ "INFLUX_HOST=${spec.host}" - , "INFLUX_TOKEN=${spec.tokenEnvName}" - , "INFLUX_ORG=${spec.org}" - , "INFLUX_BUCKET_NAME=${spec.bucket}" + -> [ "INFLUXDB_HOST=${spec.host}" + , "INFLUXDB_TOKEN=${spec.tokenEnvName}" + , "INFLUXDB_ORG=${spec.org}" + , "INFLUXDB_BUCKET_NAME=${spec.bucket}" ] let mainlineBranches = "[develop,compatible,master]" From f45728b5f62149c71456a803918d2289b6ed8fbc Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 19 Nov 2024 23:26:48 +0100 Subject: [PATCH 06/11] adjust INFLUX_* envs --- buildkite/src/Constants/Benchmarks.dhall | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/buildkite/src/Constants/Benchmarks.dhall b/buildkite/src/Constants/Benchmarks.dhall index 7747600e4d0..deae9b50173 100644 --- a/buildkite/src/Constants/Benchmarks.dhall +++ b/buildkite/src/Constants/Benchmarks.dhall @@ -10,10 +10,10 @@ let Spec = let toEnvList = \(spec : Spec.Type) - -> [ "INFLUXDB_HOST=${spec.host}" - , "INFLUXDB_TOKEN=${spec.tokenEnvName}" - , "INFLUXDB_ORG=${spec.org}" - , "INFLUXDB_BUCKET_NAME=${spec.bucket}" + -> [ "INFLUX_HOST=${spec.host}" + , "INFLUX_TOKEN=${spec.tokenEnvName}" + , "INFLUX_ORG=${spec.org}" + , "INFLUX_BUCKET_NAME=${spec.bucket}" ] let mainlineBranches = "[develop,compatible,master]" From d8779a2a0f1e175308e540af7229001ac474a653 Mon Sep 17 00:00:00 2001 From: dkijania Date: Wed, 20 Nov 2024 20:48:02 +0100 Subject: [PATCH 07/11] fix bucket name env var --- buildkite/src/Constants/Benchmarks.dhall | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/src/Constants/Benchmarks.dhall b/buildkite/src/Constants/Benchmarks.dhall index deae9b50173..d6f2703e45d 100644 --- a/buildkite/src/Constants/Benchmarks.dhall +++ b/buildkite/src/Constants/Benchmarks.dhall @@ -2,7 +2,7 @@ let Spec = { Type = { tokenEnvName : Text, bucket : Text, org : Text, host : Text } , default = { tokenEnvName = "\\\${INFLUXDB_TOKEN}" - , bucket = "\\\${INFLUXDB_BUCKET_NAME}" + , bucket = "\\\${INFLUXDB_BUCKET}" , org = "\\\${INFLUXDB_ORG}" , host = "\\\${INFLUXDB_HOST}" } From 9a423f974046080828e33a7fe5661406506fe271 Mon Sep 17 00:00:00 2001 From: dkijania Date: Wed, 20 Nov 2024 20:48:09 +0100 Subject: [PATCH 08/11] fix ledger export bench --- scripts/benchmarks/lib/bench.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/scripts/benchmarks/lib/bench.py b/scripts/benchmarks/lib/bench.py index 0034ade912f..5cea5bbb3fc 100644 --- a/scripts/benchmarks/lib/bench.py +++ b/scripts/benchmarks/lib/bench.py @@ -256,18 +256,24 @@ def export_to_csv(self, lines, filename, influxdb, branch): else: raise Exception( "Time can be expressed only in us or ns") + else: # us rows[1] = time[:-2] - # kc + + if rows[2].endswith("kc"): rows[2] = rows[2][:-2] - # w - rows[3] = rows[3][:-1] - # w - rows[4] = rows[4][:-1] - # w - rows[5] = rows[5][:-1] - rows.append(branch) + else: + #c + rows[2] = rows[2][:-1] + # w + rows[3] = rows[3][:-1] + # w + rows[4] = rows[4][:-1] + # w + rows[5] = rows[5][:-1] + + rows.append(branch) csvwriter.writerow(rows[:]) @@ -288,11 +294,12 @@ def parse(self, content, output_filename, influxdb, branch): ends = [] files = [] for i, e in enumerate(lines): - if "Running" in e: + if "Estimated testing" in e: starts.append(i) if not any(starts): self.export_to_csv(lines, output_filename, influxdb, branch) + files.append(output_filename) else: for start in starts[1:]: ends.append(start) From d19ddc1ebd3b4ce09980c32828e73d611e928354 Mon Sep 17 00:00:00 2001 From: dkijania Date: Wed, 20 Nov 2024 21:43:54 +0100 Subject: [PATCH 09/11] fix influx host if not starts with https:// --- scripts/benchmarks/lib/influx.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/benchmarks/lib/influx.py b/scripts/benchmarks/lib/influx.py index a45c1c0a39c..20b090680a8 100644 --- a/scripts/benchmarks/lib/influx.py +++ b/scripts/benchmarks/lib/influx.py @@ -90,12 +90,13 @@ def check_envs(): def client(self): Influx.check_envs() + url = os.environ[Influx.host] return influxdb_client.InfluxDBClient( - url=os.environ[Influx.host], + url= url if url.startswith('https://') else 'https://' + url, token=os.environ[Influx.token], org=os.environ[Influx.org], bucket=os.environ[Influx.bucket]) - + def __init__(self, moving_average_size=10): self.moving_average_size = moving_average_size From a617af6a964ada2d084668c154dad29e51274ddc Mon Sep 17 00:00:00 2001 From: dkijania Date: Thu, 21 Nov 2024 18:39:21 +0100 Subject: [PATCH 10/11] fix parsing for mina-bench --- scripts/benchmarks/lib/bench.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/benchmarks/lib/bench.py b/scripts/benchmarks/lib/bench.py index 5cea5bbb3fc..8fb161c7f90 100644 --- a/scripts/benchmarks/lib/bench.py +++ b/scripts/benchmarks/lib/bench.py @@ -308,7 +308,7 @@ def parse(self, content, output_filename, influxdb, branch): for start, end in zip(starts, ends): name = parse.parse('Running inline tests in library "{}"', - lines[start].strip())[0] + lines[start-1].strip())[0] file = f'{name}_{output_filename}' logger.info(f"exporting {file}..") self.export_to_csv(lines[start:end], f'{file}', influxdb, From 7473a7cd4f54452d7505bae1d5c48f7f0869b389 Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 13 Jan 2025 16:56:05 +0100 Subject: [PATCH 11/11] fix empty space after colon --- src/app/zkapp_limits/zkapp_limits.ml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app/zkapp_limits/zkapp_limits.ml b/src/app/zkapp_limits/zkapp_limits.ml index d2cf0345ba6..b2f14cfe18e 100644 --- a/src/app/zkapp_limits/zkapp_limits.ml +++ b/src/app/zkapp_limits/zkapp_limits.ml @@ -11,7 +11,7 @@ let main () = max_event_elements ; printf "max field elements for actions per transaction: %d\n" max_action_elements ; - printf "All possible zkApp account update combinations: \n" ; + printf "All possible zkApp account update combinations:\n" ; List.iter values ~f:(fun proofs -> List.iter values ~f:(fun signed_single_segments -> List.iter values ~f:(fun signed_pair_segments ->