Merge pull request #16511 from MinaProtocol/dkijania/build_performanc…

…e_tooling_in_ci_dev

[Dev] Run benchmarks in CI

buildkite/scripts/bench/run.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+set -eox pipefail
+
+# Don't prompt for answers during apt-get install
+
+YELLOW_THRESHOLD="0.1"
+RED_THRESHOLD="0.3"
+EXTRA_ARGS=""
+
+source buildkite/scripts/bench/install.sh
+
+MAINLINE_BRANCHES="-m develop -m compatile -m master -m dkijania/build_performance_tooling_in_ci"
+while [[ "$#" -gt 0 ]]; do case $1 in
+  heap-usage) BENCHMARK="heap-usage"; ;;
+  mina-base) BENCHMARK="mina-base"; ;;
+  ledger-export) 
+    BENCHMARK="ledger-export"
+    EXTRA_ARGS="--genesis-ledger-path ./genesis_ledgers/devnet.json"
+  ;;
+  snark) 
+    BENCHMARK="snark";
+    K=1
+    MAX_NUM_UPDATES=4
+    MIN_NUM_UPDATES=2
+    EXTRA_ARGS="--k ${K} --max-num-updates ${MAX_NUM_UPDATES} --min-num-updates ${MIN_NUM_UPDATES}"
+  ;;
+  zkapp) BENCHMARK="zkapp"; ;;
+  --yellow-threshold) YELLOW_THRESHOLD="$2"; shift;;
+  --red-threshold) RED_THRESHOLD="$2"; shift;;
+  *) echo "Unknown parameter passed: $1"; exit 1;;
+esac; shift; done
+
+python3 ./scripts/benchmarks test --benchmark ${BENCHMARK}  --branch ${BUILDKITE_BRANCH} --tmpfile ${BENCHMARK}.csv --yellow-threshold $YELLOW_THRESHOLD --red-threshold $RED_THRESHOLD $MAINLINE_BRANCHES $EXTRA_ARGS

buildkite/src/Command/Bench/Base.dhall

@@ -60,10 +60,11 @@ let command
             , commands =
                 RunInToolchain.runInToolchain
                   (Benchmarks.toEnvList Benchmarks.Type::{=})
-                  "./buildkite/scripts/benchmarks.sh  ${spec.bench} --red-threshold ${Double/show
-                                                                                        spec.redThreshold} --yellow-threshold ${Double/show
-                                                                                                                                  spec.yellowThreshold}"
-            , label = "Perf: ${spec.label}"
+                  "./buildkite/scripts/bench/run.sh  ${spec.bench} --red-threshold ${Double/show
+                                                                                       spec.redThreshold} --yellow-threshold ${Double/show
+                                                                                                                                 spec.yellowThreshold}"
+            , label =
+                "Perf: ${spec.label} ${PipelineMode.capitalName spec.mode}"
             , key = spec.key
             , target = spec.size
             , docker = None Docker.Type
@@ -80,8 +81,12 @@ let pipeline
                   , SelectFiles.exactly
                       "buildkite/src/Command/Bench/Base"
                       "dhall"
+                  , SelectFiles.exactly "buildkite/scripts/bench/install" "sh"
+                  , SelectFiles.exactly "buildkite/scripts/bench/run" "sh"
                   , SelectFiles.contains "scripts/benchmark"
-                  , SelectFiles.contains "buildkite/scripts/benchmark"
+                  , SelectFiles.exactly
+                      "buildkite/src/Jobs/Bench/${spec.name}"
+                      "dhall"
                   ]
                 # spec.additionalDirtyWhen
             , path = spec.path

buildkite/src/Constants/Benchmarks.dhall

@@ -1,10 +1,10 @@
 let Spec =
       { Type = { tokenEnvName : Text, bucket : Text, org : Text, host : Text }
       , default =
-          { tokenEnvName = "\\\${INFLUX_TOKEN}"
-          , bucket = "\\\${INFLUX_BUCKET_NAME}"
-          , org = "\\\${INFLUX_ORG}"
-          , host = "\\\${INFLUX_HOST}"
+          { tokenEnvName = "\\\${INFLUXDB_TOKEN}"
+          , bucket = "\\\${INFLUXDB_BUCKET}"
+          , org = "\\\${INFLUXDB_ORG}"
+          , host = "\\\${INFLUXDB_HOST}"
           }
       }
 

buildkite/src/Constants/DebianVersions.dhall

@@ -83,8 +83,6 @@ let minimalDirtyWhen =
       , S.exactly "buildkite/scripts/build-artifact" "sh"
       , S.exactly "buildkite/scripts/build-hardfork-package" "sh"
       , S.exactly "buildkite/scripts/check-compatibility" "sh"
-      , S.exactly "buildkite/src/Jobs/Test/RunSnarkProfiler" "dhall"
-      , S.exactly "buildkite/scripts/bench/snark_transaction_profiler" "sh"
       , S.exactly "buildkite/scripts/version-linter" "sh"
       , S.exactly "scripts/version-linter" "py"
       ]

buildkite/src/Jobs/Bench/HeapUsageStable.dhall

@@ -0,0 +1,21 @@
+let BenchBase = ../../Command/Bench/Base.dhall
+
+let Pipeline = ../../Pipeline/Dsl.dhall
+
+let PipelineMode = ../../Pipeline/Mode.dhall
+
+let name = "HeapUsageStable"
+
+let bench = "heap-usage"
+
+in  Pipeline.build
+      ( BenchBase.pipeline
+          BenchBase.Spec::{
+          , mode = PipelineMode.Type.Stable
+          , path = "Bench"
+          , name = name
+          , label = "Heap Usage"
+          , key = bench
+          , bench = bench
+          }
+      )

buildkite/src/Jobs/Bench/HeapUsageUnstable.dhall

@@ -0,0 +1,20 @@
+let BenchBase = ../../Command/Bench/Base.dhall
+
+let Pipeline = ../../Pipeline/Dsl.dhall
+
+let name = "HeapUsageUnstable"
+
+let bench = "heap-usage"
+
+in  Pipeline.build
+      ( BenchBase.pipeline
+          BenchBase.Spec::{
+          , path = "Bench"
+          , name = name
+          , label = "Heap Usage"
+          , key = bench
+          , bench = bench
+          , yellowThreshold = 0.1
+          , redThreshold = 0.3
+          }
+      )

buildkite/src/Jobs/Bench/LedgerExportStable.dhall

@@ -0,0 +1,21 @@
+let BenchBase = ../../Command/Bench/Base.dhall
+
+let Pipeline = ../../Pipeline/Dsl.dhall
+
+let PipelineMode = ../../Pipeline/Mode.dhall
+
+let name = "LedgerExportStable"
+
+let bench = "ledger-export"
+
+in  Pipeline.build
+      ( BenchBase.pipeline
+          BenchBase.Spec::{
+          , mode = PipelineMode.Type.Stable
+          , path = "Bench"
+          , name = name
+          , label = "Ledger Export"
+          , key = bench
+          , bench = bench
+          }
+      )

buildkite/src/Jobs/Bench/LedgerExportUnstable.dhall

@@ -0,0 +1,23 @@
+let BenchBase = ../../Command/Bench/Base.dhall
+
+let Pipeline = ../../Pipeline/Dsl.dhall
+
+let Size = ../../Command/Size.dhall
+
+let name = "LedgerExportUnstable"
+
+let bench = "ledger-export"
+
+in  Pipeline.build
+      ( BenchBase.pipeline
+          BenchBase.Spec::{
+          , path = "Bench"
+          , name = name
+          , label = "Ledger Export"
+          , size = Size.Small
+          , bench = bench
+          , key = bench
+          , yellowThreshold = 0.1
+          , redThreshold = 0.3
+          }
+      )

buildkite/src/Jobs/Bench/MinaBaseStable.dhall

@@ -0,0 +1,21 @@
+let BenchBase = ../../Command/Bench/Base.dhall
+
+let Pipeline = ../../Pipeline/Dsl.dhall
+
+let PipelineMode = ../../Pipeline/Mode.dhall
+
+let name = "MinaBaseStable"
+
+let bench = "mina-base"
+
+in  Pipeline.build
+      ( BenchBase.pipeline
+          BenchBase.Spec::{
+          , mode = PipelineMode.Type.Stable
+          , path = "Bench"
+          , name = name
+          , label = "Mina Base"
+          , key = bench
+          , bench = bench
+          }
+      )

buildkite/src/Jobs/Bench/MinaBaseUnstable.dhall

@@ -0,0 +1,23 @@
+let BenchBase = ../../Command/Bench/Base.dhall
+
+let Pipeline = ../../Pipeline/Dsl.dhall
+
+let Size = ../../Command/Size.dhall
+
+let name = "MinaBaseUnstable"
+
+let bench = "mina-base"
+
+in  Pipeline.build
+      ( BenchBase.pipeline
+          BenchBase.Spec::{
+          , path = "Bench"
+          , name = name
+          , label = "Mina Base"
+          , size = Size.Small
+          , bench = bench
+          , key = bench
+          , yellowThreshold = 0.1
+          , redThreshold = 0.3
+          }
+      )

buildkite/src/Jobs/Bench/SnarkProfilerStable.dhall

@@ -0,0 +1,21 @@
+let BenchBase = ../../Command/Bench/Base.dhall
+
+let Pipeline = ../../Pipeline/Dsl.dhall
+
+let PipelineMode = ../../Pipeline/Mode.dhall
+
+let name = "SnarkProfilerStable"
+
+let bench = "snark"
+
+in  Pipeline.build
+      ( BenchBase.pipeline
+          BenchBase.Spec::{
+          , mode = PipelineMode.Type.Stable
+          , path = "Bench"
+          , name = name
+          , label = "Snark Profiler"
+          , key = bench
+          , bench = bench
+          }
+      )

buildkite/src/Jobs/Bench/SnarkProfilerUnstable.dhall

@@ -0,0 +1,23 @@
+let BenchBase = ../../Command/Bench/Base.dhall
+
+let Pipeline = ../../Pipeline/Dsl.dhall
+
+let Size = ../../Command/Size.dhall
+
+let name = "SnarkProfilerUnstable"
+
+let bench = "snark"
+
+in  Pipeline.build
+      ( BenchBase.pipeline
+          BenchBase.Spec::{
+          , path = "Bench"
+          , name = name
+          , label = "Snark Profiler"
+          , size = Size.Small
+          , bench = bench
+          , key = bench
+          , yellowThreshold = 0.1
+          , redThreshold = 0.3
+          }
+      )

buildkite/src/Jobs/Bench/ZkappLimitsStable.dhall

@@ -0,0 +1,21 @@
+let BenchBase = ../../Command/Bench/Base.dhall
+
+let Pipeline = ../../Pipeline/Dsl.dhall
+
+let PipelineMode = ../../Pipeline/Mode.dhall
+
+let name = "ZkappLimitsStable"
+
+let bench = "zkapp"
+
+in  Pipeline.build
+      ( BenchBase.pipeline
+          BenchBase.Spec::{
+          , mode = PipelineMode.Type.Stable
+          , path = "Bench"
+          , name = name
+          , label = "Zkapp Limits"
+          , key = bench
+          , bench = bench
+          }
+      )

buildkite/src/Jobs/Bench/ZkappLimitsUnstable.dhall

@@ -0,0 +1,23 @@
+let BenchBase = ../../Command/Bench/Base.dhall
+
+let Pipeline = ../../Pipeline/Dsl.dhall
+
+let Size = ../../Command/Size.dhall
+
+let name = "ZkappLimitsUnstable"
+
+let bench = "zkapp"
+
+in  Pipeline.build
+      ( BenchBase.pipeline
+          BenchBase.Spec::{
+          , path = "Bench"
+          , name = name
+          , label = "Zkapp Limits"
+          , size = Size.Small
+          , key = bench
+          , bench = bench
+          , yellowThreshold = 0.1
+          , redThreshold = 0.3
+          }
+      )