From 5dfb734f263be9ab217c28cdf04b9e5588f4950a Mon Sep 17 00:00:00 2001 From: dkijania Date: Wed, 27 Sep 2023 12:53:50 +0200 Subject: [PATCH 01/32] run only fast checks in rosetta int tests --- .../scripts/rosetta-integration-tests.sh | 46 ++++++++++++++----- .../Jobs/Test/RosettaIntegrationTests.dhall | 2 +- 2 files changed, 36 insertions(+), 12 deletions(-) diff --git a/buildkite/scripts/rosetta-integration-tests.sh b/buildkite/scripts/rosetta-integration-tests.sh index 1efca1562ef..69855fc5a6a 100755 --- a/buildkite/scripts/rosetta-integration-tests.sh +++ b/buildkite/scripts/rosetta-integration-tests.sh @@ -17,6 +17,19 @@ set -eo pipefail +if [[ $# -ne 1 ]]; then + echo "Usage: $0 " + echo " test-stages-to-run (int): Number of test stages to run" + echo " 1 - VALIDATE CONF FILE" + echo " 2 - CHECK:SPEC" + echo " 3 - CHECK:CONSTRUCTION and above" + echo " 4 - CHECK:DATA and above" + echo " 5 - CHECK:PERF and above" + exit 1 +fi + +STAGES_TO_RUN=$1 + export MINA_NETWORK=${MINA_NETWORK:=sandbox} export LOG_LEVEL="${LOG_LEVEL:=Info}" @@ -320,18 +333,29 @@ sleep_time=$((($next_block_time - $curr_time) / 1000)) echo "Sleeping for ${sleep_time}s until next block is created..." sleep ${sleep_time} -# Mina Rosetta Checks (spec construction data perf) -echo "============ ROSETTA CLI: VALIDATE CONF FILE ${ROSETTA_CONFIGURATION_FILE} ==============" -rosetta-cli configuration:validate ${ROSETTA_CONFIGURATION_FILE} -echo "========================= ROSETTA CLI: CHECK:SPEC ===========================" -rosetta-cli check:spec --all --configuration-file ${ROSETTA_CONFIGURATION_FILE} +if [[ $STAGES_TO_RUN -gt 0 ]]; then + # Mina Rosetta Checks (spec construction data perf) + echo "============ ROSETTA CLI: VALIDATE CONF FILE ${ROSETTA_CONFIGURATION_FILE} ==============" + rosetta-cli configuration:validate ${ROSETTA_CONFIGURATION_FILE} +fi + +if [[ $STAGES_TO_RUN -gt 1 ]]; then + echo "========================= ROSETTA CLI: CHECK:SPEC ===========================" + rosetta-cli check:spec --all --configuration-file ${ROSETTA_CONFIGURATION_FILE} +fi -echo "========================= ROSETTA CLI: CHECK:CONSTRUCTION ===========================" -rosetta-cli check:construction --configuration-file ${ROSETTA_CONFIGURATION_FILE} +if [[ $STAGES_TO_RUN -gt 2 ]]; then + echo "========================= ROSETTA CLI: CHECK:CONSTRUCTION ===========================" + rosetta-cli check:construction --configuration-file ${ROSETTA_CONFIGURATION_FILE} +fi -echo "========================= ROSETTA CLI: CHECK:DATA ===========================" -rosetta-cli check:data --configuration-file ${ROSETTA_CONFIGURATION_FILE} +if [[ $STAGES_TO_RUN -gt 3 ]]; then + echo "========================= ROSETTA CLI: CHECK:DATA ===========================" + rosetta-cli check:data --configuration-file ${ROSETTA_CONFIGURATION_FILE} +fi -echo "========================= ROSETTA CLI: CHECK:PERF ===========================" -echo "rosetta-cli check:perf" # Will run this command when tests are fully implemented +if [[ $STAGES_TO_RUN -gt 4 ]]; then + echo "========================= ROSETTA CLI: CHECK:PERF ===========================" + echo "rosetta-cli check:perf" # Will run this command when tests are fully implemented +fi diff --git a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall index 4b8d0c73302..3f01fd374ce 100644 --- a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall +++ b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall @@ -39,7 +39,7 @@ Pipeline.build Command.Config::{ commands = [ Cmd.run ("export MINA_DEB_CODENAME=bullseye && source ./buildkite/scripts/export-git-env-vars.sh && echo \\\${MINA_DOCKER_TAG}"), - Cmd.runInDocker Cmd.Docker::{image="gcr.io/o1labs-192920/mina-rosetta:\\\${MINA_DOCKER_TAG}", entrypoint=" --entrypoint buildkite/scripts/rosetta-integration-tests.sh"} "bash" + Cmd.runInDocker Cmd.Docker::{image="gcr.io/o1labs-192920/mina-rosetta:\\\${MINA_DOCKER_TAG}", entrypoint=" --entrypoint 'buildkite/scripts/rosetta-integration-tests_quick.sh 1'"} "bash" ], label = "Rosetta integration tests Bullseye" , key = "rosetta-integration-tests-bullseye" From 6118445c19d2cb8a466884d23ca485db36bcb205 Mon Sep 17 00:00:00 2001 From: dkijania Date: Wed, 27 Sep 2023 19:15:56 +0200 Subject: [PATCH 02/32] add fast script for rosetta --- buildkite/scripts/rosetta-integration-tests-fast.sh | 1 + buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100755 buildkite/scripts/rosetta-integration-tests-fast.sh diff --git a/buildkite/scripts/rosetta-integration-tests-fast.sh b/buildkite/scripts/rosetta-integration-tests-fast.sh new file mode 100755 index 00000000000..f34488f4a91 --- /dev/null +++ b/buildkite/scripts/rosetta-integration-tests-fast.sh @@ -0,0 +1 @@ +./rosetta-integration-tests.sh 1 \ No newline at end of file diff --git a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall index 3f01fd374ce..6c24cf38785 100644 --- a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall +++ b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall @@ -39,7 +39,7 @@ Pipeline.build Command.Config::{ commands = [ Cmd.run ("export MINA_DEB_CODENAME=bullseye && source ./buildkite/scripts/export-git-env-vars.sh && echo \\\${MINA_DOCKER_TAG}"), - Cmd.runInDocker Cmd.Docker::{image="gcr.io/o1labs-192920/mina-rosetta:\\\${MINA_DOCKER_TAG}", entrypoint=" --entrypoint 'buildkite/scripts/rosetta-integration-tests_quick.sh 1'"} "bash" + Cmd.runInDocker Cmd.Docker::{image="gcr.io/o1labs-192920/mina-rosetta:\\\${MINA_DOCKER_TAG}", entrypoint=" --entrypoint buildkite/scripts/rosetta-integration-tests-fast.sh"} "bash" ], label = "Rosetta integration tests Bullseye" , key = "rosetta-integration-tests-bullseye" From 1d89857919eb191286f931e7949bc4d5adfd7e58 Mon Sep 17 00:00:00 2001 From: dkijania Date: Thu, 28 Sep 2023 08:56:27 +0200 Subject: [PATCH 03/32] correct path --- buildkite/scripts/rosetta-integration-tests-fast.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/scripts/rosetta-integration-tests-fast.sh b/buildkite/scripts/rosetta-integration-tests-fast.sh index f34488f4a91..311c8987389 100755 --- a/buildkite/scripts/rosetta-integration-tests-fast.sh +++ b/buildkite/scripts/rosetta-integration-tests-fast.sh @@ -1 +1 @@ -./rosetta-integration-tests.sh 1 \ No newline at end of file +buildkite/scripts/rosetta-integration-tests/rosetta-integration-tests.sh 1 \ No newline at end of file From 287fdc4a14a276a59046edb383a932a6cdbb1ec4 Mon Sep 17 00:00:00 2001 From: Dariusz Kijania Date: Thu, 28 Sep 2023 20:25:56 +0200 Subject: [PATCH 04/32] Update rosetta-integration-tests-fast.sh --- buildkite/scripts/rosetta-integration-tests-fast.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/scripts/rosetta-integration-tests-fast.sh b/buildkite/scripts/rosetta-integration-tests-fast.sh index 311c8987389..c3463e86649 100755 --- a/buildkite/scripts/rosetta-integration-tests-fast.sh +++ b/buildkite/scripts/rosetta-integration-tests-fast.sh @@ -1 +1 @@ -buildkite/scripts/rosetta-integration-tests/rosetta-integration-tests.sh 1 \ No newline at end of file +buildkite/scripts/rosetta-integration-tests.sh 1 From 210921068d158250d993c94d955c18ddb8f8d275 Mon Sep 17 00:00:00 2001 From: dkijania Date: Sat, 30 Sep 2023 13:48:14 +0200 Subject: [PATCH 05/32] more validations --- buildkite/scripts/rosetta-integration-tests-fast.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/scripts/rosetta-integration-tests-fast.sh b/buildkite/scripts/rosetta-integration-tests-fast.sh index c3463e86649..b9026923232 100755 --- a/buildkite/scripts/rosetta-integration-tests-fast.sh +++ b/buildkite/scripts/rosetta-integration-tests-fast.sh @@ -1 +1 @@ -buildkite/scripts/rosetta-integration-tests.sh 1 +buildkite/scripts/rosetta-integration-tests.sh 2 From 591b5bfb040d0d82c858b1ae39d2f34cf5102828 Mon Sep 17 00:00:00 2001 From: dkijania Date: Sat, 30 Sep 2023 14:51:25 +0200 Subject: [PATCH 06/32] update counte --- buildkite/scripts/rosetta-integration-tests-fast.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/scripts/rosetta-integration-tests-fast.sh b/buildkite/scripts/rosetta-integration-tests-fast.sh index b9026923232..619a900bd13 100755 --- a/buildkite/scripts/rosetta-integration-tests-fast.sh +++ b/buildkite/scripts/rosetta-integration-tests-fast.sh @@ -1 +1 @@ -buildkite/scripts/rosetta-integration-tests.sh 2 +buildkite/scripts/rosetta-integration-tests.sh 3 From 536d242c2116feb04bafc8e05392017e22a3caa0 Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 2 Oct 2023 09:34:22 +0200 Subject: [PATCH 07/32] Revert "Auxiliary commit to revert individual files from 591b5bfb040d0d82c858b1ae39d2f34cf5102828" This reverts commit 75f49e0c09faed96c13978db6b390acedcdb3756. --- buildkite/scripts/rosetta-integration-tests-fast.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/scripts/rosetta-integration-tests-fast.sh b/buildkite/scripts/rosetta-integration-tests-fast.sh index 619a900bd13..b9026923232 100755 --- a/buildkite/scripts/rosetta-integration-tests-fast.sh +++ b/buildkite/scripts/rosetta-integration-tests-fast.sh @@ -1 +1 @@ -buildkite/scripts/rosetta-integration-tests.sh 3 +buildkite/scripts/rosetta-integration-tests.sh 2 From 4066dc1b10abc224438501727841aa75cca182be Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 2 Oct 2023 14:32:49 +0200 Subject: [PATCH 08/32] add quick test with minimal checks and no zkapps use for pull request checks --- .../scripts/rosetta-integration-tests-fast.sh | 2 +- .../scripts/rosetta-integration-tests-full.sh | 1 + .../scripts/rosetta-integration-tests.sh | 176 +++++++++--------- .../Jobs/Test/RosettaIntegrationTests.dhall | 1 + .../Test/RosettaIntegrationTestsLong.dhall | 53 ++++++ 5 files changed, 149 insertions(+), 84 deletions(-) create mode 100755 buildkite/scripts/rosetta-integration-tests-full.sh create mode 100644 buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall diff --git a/buildkite/scripts/rosetta-integration-tests-fast.sh b/buildkite/scripts/rosetta-integration-tests-fast.sh index b9026923232..a548163395f 100755 --- a/buildkite/scripts/rosetta-integration-tests-fast.sh +++ b/buildkite/scripts/rosetta-integration-tests-fast.sh @@ -1 +1 @@ -buildkite/scripts/rosetta-integration-tests.sh 2 +buildkite/scripts/rosetta-integration-tests.sh --mode=minimal diff --git a/buildkite/scripts/rosetta-integration-tests-full.sh b/buildkite/scripts/rosetta-integration-tests-full.sh new file mode 100755 index 00000000000..49ecf784eb1 --- /dev/null +++ b/buildkite/scripts/rosetta-integration-tests-full.sh @@ -0,0 +1 @@ +buildkite/scripts/rosetta-integration-tests.sh --with-zkapps --mode=full diff --git a/buildkite/scripts/rosetta-integration-tests.sh b/buildkite/scripts/rosetta-integration-tests.sh index 69855fc5a6a..85f431b1892 100755 --- a/buildkite/scripts/rosetta-integration-tests.sh +++ b/buildkite/scripts/rosetta-integration-tests.sh @@ -16,19 +16,27 @@ # See https://github.com/coinbase/rosetta-sdk-go/blob/master/keys/signer_pallas.go#L222 set -eo pipefail - -if [[ $# -ne 1 ]]; then - echo "Usage: $0 " - echo " test-stages-to-run (int): Number of test stages to run" - echo " 1 - VALIDATE CONF FILE" - echo " 2 - CHECK:SPEC" - echo " 3 - CHECK:CONSTRUCTION and above" - echo " 4 - CHECK:DATA and above" - echo " 5 - CHECK:PERF and above" - exit 1 -fi - -STAGES_TO_RUN=$1 +#! /bin/bash + +# If yes then script will use zkapps operations when testing rosetta +ENABLE_ZKAPPS_OPS=0 + +# Defines scope of test. Currently supported are: +# - minimal -> only quick checks (~5 mins) +# - full -> all checks +MODE="minimal" + +while [ $# -gt 0 ]; do + case "$1" in + --with-zkapps) + ENABLE_ZKAPPS_OPS=1 + ;; + --mode=*) + MODE="${1#*=}" + ;; + esac + shift +done export MINA_NETWORK=${MINA_NETWORK:=sandbox} export LOG_LEVEL="${LOG_LEVEL:=Info}" @@ -250,6 +258,8 @@ send_payments() { } send_payments & +if [[ $STAGES_TO_RUN -gt 0 ]]; then + # Fee payer cache creation echo "==================== PREPARE FEE PAYER CACHE ======================" zkapp_fee_payer_pk=$(cat ${MINA_KEYS_PATH}/zkapp-${zkapp}-fee-payer.key.pub) @@ -258,31 +268,35 @@ zkapp_fee_payer_privkey=$(mina-dev advanced dump-keypair --privkey-path "${MINA_ mkdir -p /root/.cache/zkapp-cli/keys echo -e "{\n \"privateKey\": \"${zkapp_fee_payer_privkey}\",\n \"publicKey\": \"${zkapp_fee_payer_pk}\"\n}" >/root/.cache/zkapp-cli/keys/sandbox.json -# Deploy zkApps -echo "==================== DEPLOYING ZKAPPS ======================" -echo "If this fails, it's likely due to incompatibility between the -o1js and zkapp-cli versions in use." -echo "NOTE: At the moment the daemon still has an old version of - snarkyjs pinned to it, so we cannot use the latest version of - zkapp-cli, which requires o1js. Once the pinned snarkyjs version - gets updated, this build will most likely fail and we will then - need to update the zkapp-cli version used here. This is - unfortunate, but necessary. Please delete this warning once it's - done." > /dev/stderr - -deploy_txs=() -for zkapp_path in ${ZKAPP_PATH}/*/; do - zkapp_path=${zkapp_path%/} - zkapp=$(basename $zkapp_path) - echo "Deploying ${zkapp}..." - zkapp_account_pk=$(cat ${MINA_KEYS_PATH}/zkapp-${zkapp}-account.key.pub) - zkapp_account_privkey=$(mina-dev advanced dump-keypair --privkey-path "${MINA_KEYS_PATH}/zkapp-${zkapp}-account.key" | sed -ne "s/Private key: //p") +# if override not provided, default to testnets DIR +if [[ $ENABLE_ZKAPPS_OPS == 1 ]]; then + + # Deploy zkApps + echo "==================== DEPLOYING ZKAPPS ======================" + echo "If this fails, it's likely due to incompatibility between the + o1js and zkapp-cli versions in use." + echo "NOTE: At the moment the daemon still has an old version of + snarkyjs pinned to it, so we cannot use the latest version of + zkapp-cli, which requires o1js. Once the pinned snarkyjs version + gets updated, this build will most likely fail and we will then + need to update the zkapp-cli version used here. This is + unfortunate, but necessary. Please delete this warning once it's + done." > /dev/stderr + + deploy_txs=() + for zkapp_path in ${ZKAPP_PATH}/*/; do + zkapp_path=${zkapp_path%/} + zkapp=$(basename $zkapp_path) + echo "Deploying ${zkapp}..." + + zkapp_account_pk=$(cat ${MINA_KEYS_PATH}/zkapp-${zkapp}-account.key.pub) + zkapp_account_privkey=$(mina-dev advanced dump-keypair --privkey-path "${MINA_KEYS_PATH}/zkapp-${zkapp}-account.key" | sed -ne "s/Private key: //p") - mkdir -p ${zkapp_path}/keys - echo -e "{\n \"privateKey\": \"${zkapp_account_privkey}\",\n \"publicKey\": \"${zkapp_account_pk}\"\n}" >"${zkapp_path}/keys/sandbox.json" + mkdir -p ${zkapp_path}/keys + echo -e "{\n \"privateKey\": \"${zkapp_account_privkey}\",\n \"publicKey\": \"${zkapp_account_pk}\"\n}" >"${zkapp_path}/keys/sandbox.json" - cat <"${zkapp_path}/config.json" + cat <"${zkapp_path}/config.json" { "version": 1, "networks": { @@ -296,66 +310,62 @@ for zkapp_path in ${ZKAPP_PATH}/*/; do } } EOF - cd "$zkapp_path" - npm ci - npm run build - txn=$(zk deploy sandbox -y | sed -ne "s/https:\/\/berkeley.minaexplorer.com\/transaction\///p") - deploy_txs+=txn - cd - - echo "Done." -done - -# TODO: wait until all zkApps deploy txns are included in a block - -next_block_time=$(mina-dev client status --json | jq '.next_block_production.timing[1].time' | tr -d '"') -curr_time=$(date +%s%N | cut -b1-13) -sleep_time=$((($next_block_time - $curr_time) / 1000)) -echo "Sleeping for ${sleep_time}s until next block is created..." -sleep ${sleep_time} + cd "$zkapp_path" + npm ci + npm run build + txn=$(zk deploy sandbox -y | sed -ne "s/https:\/\/berkeley.minaexplorer.com\/transaction\///p") + deploy_txs+=txn + cd - + echo "Done." + done -# Start calling zkApp methods -echo "==================== INTERACTING WITH ZKAPPS ======================" -RECEIVER_PK=$BLOCK_PRODUCER_PK -for zkapp_path in ${ZKAPP_PATH}/*/; do - zkapp_path=${zkapp_path%/} - zkapp=$(basename $zkapp_path) - echo "Interacting with ${zkapp}..." + # TODO: wait until all zkApps deploy txns are included in a block + + next_block_time=$(mina-dev client status --json | jq '.next_block_production.timing[1].time' | tr -d '"') + curr_time=$(date +%s%N | cut -b1-13) + sleep_time=$((($next_block_time - $curr_time) / 1000)) + echo "Sleeping for ${sleep_time}s until next block is created..." + sleep ${sleep_time} + + # Start calling zkApp methods + echo "==================== INTERACTING WITH ZKAPPS ======================" + RECEIVER_PK=$BLOCK_PRODUCER_PK + for zkapp_path in ${ZKAPP_PATH}/*/; do + zkapp_path=${zkapp_path%/} + zkapp=$(basename $zkapp_path) + echo "Interacting with ${zkapp}..." + + cd "$zkapp_path" + ./interact.sh sandbox + cd - + echo "Done." + done - cd "$zkapp_path" - ./interact.sh sandbox - cd - - echo "Done." -done + next_block_time=$(mina-dev client status --json | jq '.next_block_production.timing[1].time' | tr -d '"') + curr_time=$(date +%s%N | cut -b1-13) + sleep_time=$((($next_block_time - $curr_time) / 1000)) + echo "Sleeping for ${sleep_time}s until next block is created..." + sleep ${sleep_time} +fi -next_block_time=$(mina-dev client status --json | jq '.next_block_production.timing[1].time' | tr -d '"') -curr_time=$(date +%s%N | cut -b1-13) -sleep_time=$((($next_block_time - $curr_time) / 1000)) -echo "Sleeping for ${sleep_time}s until next block is created..." -sleep ${sleep_time} +# Mina Rosetta Checks (spec construction data perf) +echo "============ ROSETTA CLI: VALIDATE CONF FILE ${ROSETTA_CONFIGURATION_FILE} ==============" +rosetta-cli configuration:validate ${ROSETTA_CONFIGURATION_FILE} +echo "========================= ROSETTA CLI: CHECK:SPEC ===========================" +rosetta-cli check:spec --all --configuration-file ${ROSETTA_CONFIGURATION_FILE} -if [[ $STAGES_TO_RUN -gt 0 ]]; then - # Mina Rosetta Checks (spec construction data perf) - echo "============ ROSETTA CLI: VALIDATE CONF FILE ${ROSETTA_CONFIGURATION_FILE} ==============" - rosetta-cli configuration:validate ${ROSETTA_CONFIGURATION_FILE} -fi -if [[ $STAGES_TO_RUN -gt 1 ]]; then - echo "========================= ROSETTA CLI: CHECK:SPEC ===========================" - rosetta-cli check:spec --all --configuration-file ${ROSETTA_CONFIGURATION_FILE} -fi +# if override not provided, default to testnets DIR +if [[ $MODE == "full" ]]; then -if [[ $STAGES_TO_RUN -gt 2 ]]; then echo "========================= ROSETTA CLI: CHECK:CONSTRUCTION ===========================" rosetta-cli check:construction --configuration-file ${ROSETTA_CONFIGURATION_FILE} -fi -if [[ $STAGES_TO_RUN -gt 3 ]]; then echo "========================= ROSETTA CLI: CHECK:DATA ===========================" rosetta-cli check:data --configuration-file ${ROSETTA_CONFIGURATION_FILE} -fi -if [[ $STAGES_TO_RUN -gt 4 ]]; then echo "========================= ROSETTA CLI: CHECK:PERF ===========================" echo "rosetta-cli check:perf" # Will run this command when tests are fully implemented -fi + +fi \ No newline at end of file diff --git a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall index 6c24cf38785..cc1f780138d 100644 --- a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall +++ b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall @@ -20,6 +20,7 @@ let dirtyWhen = [ S.strictlyStart (S.contains "src/app/archive"), S.exactly "buildkite/src/Jobs/Test/RosettaIntegrationTests" "dhall", S.exactly "buildkite/scripts/rosetta-integration-tests" "sh" + S.exactly "buildkite/scripts/rosetta-integration-tests-long.sh" "sh" ] let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type diff --git a/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall b/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall new file mode 100644 index 00000000000..376cb6a8d73 --- /dev/null +++ b/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall @@ -0,0 +1,53 @@ +let Prelude = ../../External/Prelude.dhall +let B = ../../External/Buildkite.dhall + +let Cmd = ../../Lib/Cmds.dhall +let S = ../../Lib/SelectFiles.dhall + +let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineMode = ../../Pipeline/Mode.dhall +let JobSpec = ../../Pipeline/JobSpec.dhall + +let Command = ../../Command/Base.dhall +let RunInToolchain = ../../Command/RunInToolchain.dhall +let Size = ../../Command/Size.dhall +let Libp2p = ../../Command/Libp2pHelperBuild.dhall +let DockerImage = ../../Command/DockerImage.dhall +let DebianVersions = ../../Constants/DebianVersions.dhall + +let dirtyWhen = [ + S.strictlyStart (S.contains "src/app/rosetta"), + S.strictlyStart (S.contains "src/lib"), + S.strictlyStart (S.contains "src/app/archive"), + S.exactly "buildkite/src/Jobs/Test/RosettaIntegrationTests" "dhall", + S.exactly "buildkite/scripts/rosetta-integration-tests" "sh" + S.exactly "buildkite/scripts/rosetta-integration-tests-fast.sh" "sh" +] + +let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type + +in + +Pipeline.build + Pipeline.Config:: + { spec = + JobSpec::{ + dirtyWhen = dirtyWhen, + path = "Test", + name = "RosettaIntegrationTestsLong", + mode = PipelineMode.Type.Stable + } + , steps = [ + Command.build + Command.Config::{ + commands = [ + Cmd.run ("export MINA_DEB_CODENAME=bullseye && source ./buildkite/scripts/export-git-env-vars.sh && echo \\\${MINA_DOCKER_TAG}"), + Cmd.runInDocker Cmd.Docker::{image="gcr.io/o1labs-192920/mina-rosetta:\\\${MINA_DOCKER_TAG}", entrypoint=" --entrypoint buildkite/scripts/rosetta-integration-tests-full.sh"} "bash" + ], + label = "Rosetta integration tests Bullseye Long" + , key = "rosetta-integration-tests-bullseye-long" + , target = Size.Small + , depends_on = [ { name = "MinaArtifactBullseye", key = "rosetta-bullseye-docker-image" } ] + } + ] + } \ No newline at end of file From 32e266fa029f613cc6da2381b6c96d03898c4ecc Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 2 Oct 2023 15:53:25 +0200 Subject: [PATCH 09/32] compilation fixes --- buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall | 2 +- buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall index cc1f780138d..36106915503 100644 --- a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall +++ b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall @@ -19,7 +19,7 @@ let dirtyWhen = [ S.strictlyStart (S.contains "src/lib"), S.strictlyStart (S.contains "src/app/archive"), S.exactly "buildkite/src/Jobs/Test/RosettaIntegrationTests" "dhall", - S.exactly "buildkite/scripts/rosetta-integration-tests" "sh" + S.exactly "buildkite/scripts/rosetta-integration-tests" "sh", S.exactly "buildkite/scripts/rosetta-integration-tests-long.sh" "sh" ] diff --git a/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall b/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall index 376cb6a8d73..c2af7b5be95 100644 --- a/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall +++ b/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall @@ -20,7 +20,7 @@ let dirtyWhen = [ S.strictlyStart (S.contains "src/lib"), S.strictlyStart (S.contains "src/app/archive"), S.exactly "buildkite/src/Jobs/Test/RosettaIntegrationTests" "dhall", - S.exactly "buildkite/scripts/rosetta-integration-tests" "sh" + S.exactly "buildkite/scripts/rosetta-integration-tests" "sh", S.exactly "buildkite/scripts/rosetta-integration-tests-fast.sh" "sh" ] From 356d7da6d893f9dc21ae4d04485fe4ea01edc88b Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 2 Oct 2023 16:53:39 +0200 Subject: [PATCH 10/32] remove leftovers --- buildkite/scripts/rosetta-integration-tests.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/buildkite/scripts/rosetta-integration-tests.sh b/buildkite/scripts/rosetta-integration-tests.sh index 85f431b1892..acc7a771842 100755 --- a/buildkite/scripts/rosetta-integration-tests.sh +++ b/buildkite/scripts/rosetta-integration-tests.sh @@ -258,8 +258,6 @@ send_payments() { } send_payments & -if [[ $STAGES_TO_RUN -gt 0 ]]; then - # Fee payer cache creation echo "==================== PREPARE FEE PAYER CACHE ======================" zkapp_fee_payer_pk=$(cat ${MINA_KEYS_PATH}/zkapp-${zkapp}-fee-payer.key.pub) From e4ba43ac4362f1407b3c4b228b241d6047d4381d Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 2 Oct 2023 20:04:29 +0200 Subject: [PATCH 11/32] remove comment --- buildkite/scripts/rosetta-integration-tests.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/buildkite/scripts/rosetta-integration-tests.sh b/buildkite/scripts/rosetta-integration-tests.sh index acc7a771842..a79f7fcf3bc 100755 --- a/buildkite/scripts/rosetta-integration-tests.sh +++ b/buildkite/scripts/rosetta-integration-tests.sh @@ -353,8 +353,6 @@ rosetta-cli configuration:validate ${ROSETTA_CONFIGURATION_FILE} echo "========================= ROSETTA CLI: CHECK:SPEC ===========================" rosetta-cli check:spec --all --configuration-file ${ROSETTA_CONFIGURATION_FILE} - -# if override not provided, default to testnets DIR if [[ $MODE == "full" ]]; then echo "========================= ROSETTA CLI: CHECK:CONSTRUCTION ===========================" From 528e0bfc69d1b5d3dd2425140f04b9894be9d7df Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 3 Oct 2023 20:38:55 +0200 Subject: [PATCH 12/32] remove zkapps completely --- .../scripts/rosetta-integration-tests-full.sh | 2 +- .../scripts/rosetta-integration-tests.sh | 168 +----------------- 2 files changed, 5 insertions(+), 165 deletions(-) diff --git a/buildkite/scripts/rosetta-integration-tests-full.sh b/buildkite/scripts/rosetta-integration-tests-full.sh index 49ecf784eb1..1f76975d611 100755 --- a/buildkite/scripts/rosetta-integration-tests-full.sh +++ b/buildkite/scripts/rosetta-integration-tests-full.sh @@ -1 +1 @@ -buildkite/scripts/rosetta-integration-tests.sh --with-zkapps --mode=full +buildkite/scripts/rosetta-integration-tests.sh --mode=full diff --git a/buildkite/scripts/rosetta-integration-tests.sh b/buildkite/scripts/rosetta-integration-tests.sh index a79f7fcf3bc..e671e5aa78e 100755 --- a/buildkite/scripts/rosetta-integration-tests.sh +++ b/buildkite/scripts/rosetta-integration-tests.sh @@ -1,26 +1,11 @@ #!/bin/bash -# Deploy a sandboxed Mina daemon with an archive and a Rosetta instance. -# Deploy 2 zkApps from https://github.com/MinaProtocol/rosetta-integration-test-zkapps -# to the network, interact with them and add some regular transactions. -# Then run full rosetta-cli tests against the Rosetta instance. - -# NPM and NodeJS are installed through NVM, versions are stored in environment -# variables below. Zkapp-cli is installed globally through NPM, however, to -# ensure compatibility with the daemon, we use o1js pinned in the Mina repo. -# The repo is mounted into the container at /workdir, so we can build o1js from -# that source. It is important to make sure that the zkapp-cli version installed -# is compatible with o1js version used. - # These tests use the mina-dev binary, as rosetta-cli assumes we use a testnet. # See https://github.com/coinbase/rosetta-sdk-go/blob/master/keys/signer_pallas.go#L222 set -eo pipefail #! /bin/bash -# If yes then script will use zkapps operations when testing rosetta -ENABLE_ZKAPPS_OPS=0 - # Defines scope of test. Currently supported are: # - minimal -> only quick checks (~5 mins) # - full -> all checks @@ -28,9 +13,6 @@ MODE="minimal" while [ $# -gt 0 ]; do case "$1" in - --with-zkapps) - ENABLE_ZKAPPS_OPS=1 - ;; --mode=*) MODE="${1#*=}" ;; @@ -68,43 +50,6 @@ export MINA_CONFIG_FILE=$HOME/${MINA_NETWORK}.json export MINA_CONFIG_DIR="${MINA_CONFIG_DIR:=$HOME/.mina-config}" export MINA_GRAPHQL_PORT=${MINA_GRAPHQL_PORT:=3085} -# Test variables -export ROSETTA_INT_TEST_ZKAPPS_VERSION=${ROSETTA_INT_TEST_ZKAPPS_VERSION:=rosetta-ci-tests} - -# We need a version which is compatible with o1js pinned to the Mina repo. -# Should be set to 'latest' most of the time, but occasionally we might need -# an older one. -export ZKAPP_CLI_VERSION=0.11.0 - -# Nodejs variables -export NVM_VERSION=0.39.3 -export NODE_VERSION=20.6.1 - -# zkApps variables -export ZKAPP_PATH=$HOME/zkapps - -echo "=========================== INSTALLING NPM ===========================" -curl -so- https://raw.githubusercontent.com/nvm-sh/nvm/v${NVM_VERSION}/install.sh | bash &>/dev/null -export NVM_DIR="$HOME/.nvm" -[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm -[ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion" # This loads nvm bash_completion - -nvm install $NODE_VERSION -nvm use --delete-prefix $NODE_VERSION - -mkdir ~/.npm-global -npm config set prefix '~/.npm-global' -export PATH=~/.npm-global/bin:$PATH - -# Install zkapp-cli and Typescript compiler. -npm install --no-progress --global "zkapp-cli@${ZKAPP_CLI_VERSION}" "typescript@latest" - -# Build o1js so that we can use it later. -pushd /workdir/src/lib/snarkyjs -npm ci -npm run build -popd - # Rosetta CLI variables # Files from ROSETTA_CLI_CONFIG_FILES will be read from # ROSETTA_CONFIGURATION_INPUT_DIR and some placeholders will be @@ -116,10 +61,6 @@ ROSETTA_CLI_MAIN_CONFIG_FILE=${ROSETTA_CLI_MAIN_CONFIG_FILE:="config.json"} # Frequency (in seconds) at which payment operations will be sent TRANSACTION_FREQUENCY=60 -# Fetch zkApps -curl -Ls https://github.com/MinaProtocol/rosetta-integration-test-zkapps/tarball/$ROSETTA_INT_TEST_ZKAPPS_VERSION | tar xz -C /tmp -mv /tmp/MinaProtocol-rosetta-integration-test-zkapps-* $ZKAPP_PATH - # Libp2p Keypair echo "=========================== GENERATING KEYPAIR IN ${MINA_LIBP2P_KEYPAIR_PATH} ===========================" mina-dev libp2p generate-keypair -privkey-path $MINA_LIBP2P_KEYPAIR_PATH @@ -149,19 +90,6 @@ cat <"$MINA_CONFIG_FILE" } } EOF -for zkapp_path in ${ZKAPP_PATH}/*/; do - zkapp_path=${zkapp_path%/} - zkapp=$(basename $zkapp_path) - # Generate zkApp account keypair - mina-dev advanced generate-keypair --privkey-path ${MINA_KEYS_PATH}/zkapp-${zkapp}-account.key - # Generate zkApp fee payer keypair - mina-dev advanced generate-keypair --privkey-path ${MINA_KEYS_PATH}/zkapp-${zkapp}-fee-payer.key - zkapp_fee_payer_pk=$(cat $MINA_KEYS_PATH/zkapp-${zkapp}-fee-payer.key.pub) - line="[{ \"pk\": \"${zkapp_fee_payer_pk}\", \"balance\": \"10000\", \"delegate\": null, \"sk\": null }]" - jq ".ledger.accounts |= . + ${line}" $MINA_CONFIG_FILE >${MINA_CONFIG_FILE}.tmp - mv ${MINA_CONFIG_FILE}.tmp $MINA_CONFIG_FILE -done -cat $MINA_CONFIG_FILE | jq . # Substitute placeholders in rosetta-cli configuration ROSETTA_CONFIGURATION_OUTPUT_DIR=/tmp/rosetta-cli-config @@ -181,11 +109,6 @@ done echo "==================== IMPORTING GENESIS ACCOUNTS ======================" mina-dev accounts import --privkey-path $MINA_KEYS_PATH/block-producer.key --config-directory $MINA_CONFIG_DIR mina-dev accounts import --privkey-path $MINA_KEYS_PATH/snark-producer.key --config-directory $MINA_CONFIG_DIR -for zkapp_path in ${ZKAPP_PATH}/*/; do - zkapp_path=${zkapp_path%/} - zkapp=$(basename $zkapp_path) - mina-dev accounts import --privkey-path $MINA_KEYS_PATH/zkapp-${zkapp}-fee-payer.key --config-directory $MINA_CONFIG_DIR -done # Postgres echo "========================= INITIALIZING POSTGRESQL ===========================" @@ -258,93 +181,10 @@ send_payments() { } send_payments & -# Fee payer cache creation -echo "==================== PREPARE FEE PAYER CACHE ======================" -zkapp_fee_payer_pk=$(cat ${MINA_KEYS_PATH}/zkapp-${zkapp}-fee-payer.key.pub) -zkapp_fee_payer_privkey=$(mina-dev advanced dump-keypair --privkey-path "${MINA_KEYS_PATH}/zkapp-${zkapp}-fee-payer.key" | sed -ne "s/Private key: //p") - -mkdir -p /root/.cache/zkapp-cli/keys -echo -e "{\n \"privateKey\": \"${zkapp_fee_payer_privkey}\",\n \"publicKey\": \"${zkapp_fee_payer_pk}\"\n}" >/root/.cache/zkapp-cli/keys/sandbox.json - - -# if override not provided, default to testnets DIR -if [[ $ENABLE_ZKAPPS_OPS == 1 ]]; then - - # Deploy zkApps - echo "==================== DEPLOYING ZKAPPS ======================" - echo "If this fails, it's likely due to incompatibility between the - o1js and zkapp-cli versions in use." - echo "NOTE: At the moment the daemon still has an old version of - snarkyjs pinned to it, so we cannot use the latest version of - zkapp-cli, which requires o1js. Once the pinned snarkyjs version - gets updated, this build will most likely fail and we will then - need to update the zkapp-cli version used here. This is - unfortunate, but necessary. Please delete this warning once it's - done." > /dev/stderr - - deploy_txs=() - for zkapp_path in ${ZKAPP_PATH}/*/; do - zkapp_path=${zkapp_path%/} - zkapp=$(basename $zkapp_path) - echo "Deploying ${zkapp}..." - - zkapp_account_pk=$(cat ${MINA_KEYS_PATH}/zkapp-${zkapp}-account.key.pub) - zkapp_account_privkey=$(mina-dev advanced dump-keypair --privkey-path "${MINA_KEYS_PATH}/zkapp-${zkapp}-account.key" | sed -ne "s/Private key: //p") - - mkdir -p ${zkapp_path}/keys - echo -e "{\n \"privateKey\": \"${zkapp_account_privkey}\",\n \"publicKey\": \"${zkapp_account_pk}\"\n}" >"${zkapp_path}/keys/sandbox.json" - - cat <"${zkapp_path}/config.json" -{ - "version": 1, - "networks": { - "sandbox": { - "url": "http://127.0.0.1:${MINA_GRAPHQL_PORT}/graphql", - "keyPath": "keys/sandbox.json", - "feepayerKeyPath": "/root/.cache/zkapp-cli/keys/sandbox.json", - "feepayerAlias": "sandbox", - "fee": "1" - } - } -} -EOF - cd "$zkapp_path" - npm ci - npm run build - txn=$(zk deploy sandbox -y | sed -ne "s/https:\/\/berkeley.minaexplorer.com\/transaction\///p") - deploy_txs+=txn - cd - - echo "Done." - done - - # TODO: wait until all zkApps deploy txns are included in a block - - next_block_time=$(mina-dev client status --json | jq '.next_block_production.timing[1].time' | tr -d '"') - curr_time=$(date +%s%N | cut -b1-13) - sleep_time=$((($next_block_time - $curr_time) / 1000)) - echo "Sleeping for ${sleep_time}s until next block is created..." - sleep ${sleep_time} - - # Start calling zkApp methods - echo "==================== INTERACTING WITH ZKAPPS ======================" - RECEIVER_PK=$BLOCK_PRODUCER_PK - for zkapp_path in ${ZKAPP_PATH}/*/; do - zkapp_path=${zkapp_path%/} - zkapp=$(basename $zkapp_path) - echo "Interacting with ${zkapp}..." - - cd "$zkapp_path" - ./interact.sh sandbox - cd - - echo "Done." - done - - next_block_time=$(mina-dev client status --json | jq '.next_block_production.timing[1].time' | tr -d '"') - curr_time=$(date +%s%N | cut -b1-13) - sleep_time=$((($next_block_time - $curr_time) / 1000)) - echo "Sleeping for ${sleep_time}s until next block is created..." - sleep ${sleep_time} -fi +next_block_time=$(mina-dev client status --json | jq '.next_block_production.timing[1].time' | tr -d '"') curr_time=$(date +%s%N | cut -b1-13) +sleep_time=$((($next_block_time - $curr_time) / 1000)) +echo "Sleeping for ${sleep_time}s until next block is created..." +sleep ${sleep_time} # Mina Rosetta Checks (spec construction data perf) echo "============ ROSETTA CLI: VALIDATE CONF FILE ${ROSETTA_CONFIGURATION_FILE} ==============" From daeaf73b7ccb02096ea2357e8d86cc72a7032495 Mon Sep 17 00:00:00 2001 From: Serhii Shymkiv Date: Thu, 5 Oct 2023 11:51:44 +0300 Subject: [PATCH 13/32] Use zkapp_test_transaction for Rosetta integration tests (debug attempt). --- .../scripts/rosetta-integration-tests.sh | 66 +++++++++++++++++-- dockerfiles/stages/3-builder | 29 ++++---- 2 files changed, 75 insertions(+), 20 deletions(-) diff --git a/buildkite/scripts/rosetta-integration-tests.sh b/buildkite/scripts/rosetta-integration-tests.sh index e671e5aa78e..c5451b5252a 100755 --- a/buildkite/scripts/rosetta-integration-tests.sh +++ b/buildkite/scripts/rosetta-integration-tests.sh @@ -13,9 +13,9 @@ MODE="minimal" while [ $# -gt 0 ]; do case "$1" in - --mode=*) - MODE="${1#*=}" - ;; + --mode=*) + MODE="${1#*=}" + ;; esac shift done @@ -70,9 +70,18 @@ echo "=========================== GENERATING GENESIS LEDGER FOR ${MINA_NETWORK} mkdir -p $MINA_KEYS_PATH mina-dev advanced generate-keypair --privkey-path $MINA_KEYS_PATH/block-producer.key mina-dev advanced generate-keypair --privkey-path $MINA_KEYS_PATH/snark-producer.key +mina-dev advanced generate-keypair --privkey-path $MINA_KEYS_PATH/zkapp-fee-payer.key +mina-dev advanced generate-keypair --privkey-path $MINA_KEYS_PATH/zkapp-sender.key +mina-dev advanced generate-keypair --privkey-path $MINA_KEYS_PATH/zkapp-account.key chmod -R 0700 $MINA_KEYS_PATH BLOCK_PRODUCER_PK=$(cat $MINA_KEYS_PATH/block-producer.key.pub) SNARK_PRODUCER_PK=$(cat $MINA_KEYS_PATH/snark-producer.key.pub) +ZKAPP_FEE_PAYER_KEY=$MINA_KEYS_PATH/zkapp-fee-payer.key +ZKAPP_FEE_PAYER_PUB_KEY=$(cat ${ZKAPP_FEE_PAYER_KEY}.pub) +ZKAPP_SENDER_KEY=$MINA_KEYS_PATH/zkapp-sender.key +ZKAPP_SENDER_PUB_KEY=$(cat ${ZKAPP_SENDER_KEY}.pub) +ZKAPP_ACCOUNT_KEY=$MINA_KEYS_PATH/zkapp-account.key +ZKAPP_ACCOUNT_PUB_KEY=$(cat ${ZKAPP_ACCOUNT_KEY}.key.pub) mkdir -p $MINA_CONFIG_DIR/wallets/store cp $MINA_KEYS_PATH/block-producer.key $MINA_CONFIG_DIR/wallets/store/$BLOCK_PRODUCER_PK @@ -84,8 +93,11 @@ cat <"$MINA_CONFIG_FILE" "ledger": { "name": "${MINA_NETWORK}", "accounts": [ - { "pk": "${BLOCK_PRODUCER_PK}", "balance": "1000", "delegate": null, "sk": null }, - { "pk": "${SNARK_PRODUCER_PK}", "balance": "2000", "delegate": "${BLOCK_PRODUCER_PK}", "sk": null } + { "pk": "${BLOCK_PRODUCER_PK}", "balance": "11550000.000000000", "delegate": null, "sk": null }, + { "pk": "${SNARK_PRODUCER_PK}", "balance": "65500.000000000", "delegate": "${BLOCK_PRODUCER_PK}", "sk": null } + { "pk": "${ZKAPP_FEE_PAYER_PUB_KEY}", "balance": "155.000000000", "delegate": null, "sk": null }, + { "pk": "${ZKAPP_SENDER_PUB_KEY}", "balance": "155.000000000", "delegate": null, "sk": null }, + { "pk": "${ZKAPP_ACCOUNT_PUB_KEY}", "balance": "155.000000000", "delegate": null, "sk": null }, ] } } @@ -151,6 +163,7 @@ mina-dev daemon \ --libp2p-keypair ${MINA_LIBP2P_KEYPAIR_PATH} \ --log-level ${LOG_LEVEL} \ --proof-level none \ + --insecure-rest-server \ --rest-port ${MINA_GRAPHQL_PORT} \ --run-snark-worker "$SNARK_PRODUCER_PK" \ --seed \ @@ -166,6 +179,25 @@ until [ $daemon_status == "Synced" ]; do echo "Daemon Status: ${daemon_status}" done +debug_zkapp_output() { + echo "" + echo "" + echo "=========== DEBUG ZKAPP TXN (Start) ===========" + echo "" + echo "" + echo ${1} + echo "" + echo "" + echo "=========== DEBUG ZKAPP TXN (End) ===========" + echo "" + echo "" +} + +echo "========================= ZKAPP ACCOUNT SETTING UP ===========================" +QUERY=$(zkapp_test_transaction create-zkapp-account --fee-payer-key ${ZKAPP_FEE_PAYER_KEY} --nonce 0 --sender-key ${ZKAPP_SENDER_KEY} --sender-nonce 0 --receiver-amount 1000 --zkapp-account-key ${ZKAPP_ACCOUNT_KEY} --fee 5 | sed 1,7d) +# TODO: Send txn against "http://127.0.0.1:${MINA_GRAPHQL_PORT}/graphql" +debug_zkapp_output "${QUERY}" + # Unlock Genesis Accounts echo "==================== UNLOCKING GENESIS ACCOUNTS ======================" mina-dev accounts unlock --public-key $BLOCK_PRODUCER_PK @@ -181,7 +213,27 @@ send_payments() { } send_payments & -next_block_time=$(mina-dev client status --json | jq '.next_block_production.timing[1].time' | tr -d '"') curr_time=$(date +%s%N | cut -b1-13) +ZKAPP_FEE_PAYER_NONCE=1 +ZKAPP_SENDER_NONCE=1 +ZKAPP_STATE=0 +send_zkapp_transactions() { + while true; do + QUERY=$(zkapp_test_transaction transfer-funds-one-receiver --fee-payer-key ${ZKAPP_FEE_PAYER_KEY} --nonce ${ZKAPP_FEE_PAYER_NONCE} --sender-key ${ZKAPP_SENDER_KEY} --sender-nonce ${ZKAPP_SENDER_NONCE} --receiver-amount 1 --fee 5 --receiver ${ZKAPP_ACCOUNT_PUB_KEY} | sed 1,5d) + # TODO: Send txn against "http://127.0.0.1:${MINA_GRAPHQL_PORT}/graphql" + debug_zkapp_output "${QUERY}" + let ZKAPP_FEE_PAYER_NONCE++ + let ZKAPP_SENDER_NONCE++ + + QUERY=$(zkapp_test_transaction update-state --fee-payer-key ${ZKAPP_FEE_PAYER_KEY} --nonce ${ZKAPP_FEE_PAYER_NONCE} --zkapp-account-key ${ZKAPP_SENDER_KEY} --zkapp-state ${ZKAPP_STATE} --fee 5 | sed 1,5d) + # TODO: Send txn against "http://127.0.0.1:${MINA_GRAPHQL_PORT}/graphql" + debug_zkapp_output "${QUERY}" + let ZKAPP_FEE_PAYER_NONCE++ + let ZKAPP_STATE++ + done +} +send_zkapp_transactions & + +next_block_time=$(mina-dev client status --json | jq '.next_block_production.timing[1].time' | tr -d '"') curr_time=$(date +%s%N | cut -b1-13) sleep_time=$((($next_block_time - $curr_time) / 1000)) echo "Sleeping for ${sleep_time}s until next block is created..." sleep ${sleep_time} @@ -204,4 +256,4 @@ if [[ $MODE == "full" ]]; then echo "========================= ROSETTA CLI: CHECK:PERF ===========================" echo "rosetta-cli check:perf" # Will run this command when tests are fully implemented -fi \ No newline at end of file +fi diff --git a/dockerfiles/stages/3-builder b/dockerfiles/stages/3-builder index 38b36f5d066..1649a5d9e12 100644 --- a/dockerfiles/stages/3-builder +++ b/dockerfiles/stages/3-builder @@ -36,23 +36,25 @@ RUN mkdir ${HOME}/app # --- Build libp2p_helper RUN make libp2p_helper \ - && mv src/app/libp2p_helper/result/bin/libp2p_helper ${HOME}/app/libp2p_helper + && mv src/app/libp2p_helper/result/bin/libp2p_helper ${HOME}/app/libp2p_helper # --- Make rosetta-crucial components and the generate_keypair tool +# ---- Rosetta tests also need zkapp_test_transaction RUN eval $(opam config env) \ && dune build --profile=${DUNE_PROFILE} \ - src/app/cli/src/mina_testnet_signatures.exe \ - src/app/cli/src/mina_mainnet_signatures.exe \ - src/app/archive/archive.exe \ - src/app/archive_blocks/archive_blocks.exe \ - src/app/extract_blocks/extract_blocks.exe \ - src/app/missing_blocks_auditor/missing_blocks_auditor.exe \ - src/app/replayer/replayer.exe \ - src/app/rosetta/rosetta_testnet_signatures.exe \ - src/app/rosetta/rosetta_mainnet_signatures.exe \ - src/app/generate_keypair/generate_keypair.exe \ - src/app/validate_keypair/validate_keypair.exe \ - src/app/rosetta/ocaml-signer/signer.exe \ + src/app/cli/src/mina_testnet_signatures.exe \ + src/app/cli/src/mina_mainnet_signatures.exe \ + src/app/archive/archive.exe \ + src/app/archive_blocks/archive_blocks.exe \ + src/app/extract_blocks/extract_blocks.exe \ + src/app/missing_blocks_auditor/missing_blocks_auditor.exe \ + src/app/replayer/replayer.exe \ + src/app/rosetta/rosetta_testnet_signatures.exe \ + src/app/rosetta/rosetta_mainnet_signatures.exe \ + src/app/generate_keypair/generate_keypair.exe \ + src/app/validate_keypair/validate_keypair.exe \ + src/app/rosetta/ocaml-signer/signer.exe \ + src/app/zkapp_test_transaction/zkapp_test_transaction.exe \ && cp _build/default/src/app/archive_blocks/archive_blocks.exe $HOME/app/mina-archive-blocks \ && cp _build/default/src/app/extract_blocks/extract_blocks.exe $HOME/app/mina-extract-blocks \ && cp _build/default/src/app/missing_blocks_auditor/missing_blocks_auditor.exe $HOME/app/mina-missing-blocks-auditor \ @@ -65,6 +67,7 @@ RUN eval $(opam config env) \ && mv _build/default/src/app/generate_keypair/generate_keypair.exe $HOME/app/mina-generate-keypair \ && mv _build/default/src/app/validate_keypair/validate_keypair.exe $HOME/app/mina-validate-keypair \ && mv _build/default/src/app/rosetta/ocaml-signer/signer.exe $HOME/app/mina-ocaml-signer \ + && mv _build/default/src/app/zkapp_test_transaction/zkapp_test_transaction.exe $HOME/app/zkapp_test_transaction \ && rm -rf _build # --- Clear go module caches to make the container smaller From d850951926697c493b9d430a8d02b32179eada41 Mon Sep 17 00:00:00 2001 From: Serhii Shymkiv Date: Thu, 5 Oct 2023 12:41:21 +0300 Subject: [PATCH 14/32] Typo. --- buildkite/scripts/rosetta-integration-tests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/buildkite/scripts/rosetta-integration-tests.sh b/buildkite/scripts/rosetta-integration-tests.sh index c5451b5252a..f6759f304f7 100755 --- a/buildkite/scripts/rosetta-integration-tests.sh +++ b/buildkite/scripts/rosetta-integration-tests.sh @@ -81,7 +81,7 @@ ZKAPP_FEE_PAYER_PUB_KEY=$(cat ${ZKAPP_FEE_PAYER_KEY}.pub) ZKAPP_SENDER_KEY=$MINA_KEYS_PATH/zkapp-sender.key ZKAPP_SENDER_PUB_KEY=$(cat ${ZKAPP_SENDER_KEY}.pub) ZKAPP_ACCOUNT_KEY=$MINA_KEYS_PATH/zkapp-account.key -ZKAPP_ACCOUNT_PUB_KEY=$(cat ${ZKAPP_ACCOUNT_KEY}.key.pub) +ZKAPP_ACCOUNT_PUB_KEY=$(cat ${ZKAPP_ACCOUNT_KEY}.pub) mkdir -p $MINA_CONFIG_DIR/wallets/store cp $MINA_KEYS_PATH/block-producer.key $MINA_CONFIG_DIR/wallets/store/$BLOCK_PRODUCER_PK @@ -97,7 +97,7 @@ cat <"$MINA_CONFIG_FILE" { "pk": "${SNARK_PRODUCER_PK}", "balance": "65500.000000000", "delegate": "${BLOCK_PRODUCER_PK}", "sk": null } { "pk": "${ZKAPP_FEE_PAYER_PUB_KEY}", "balance": "155.000000000", "delegate": null, "sk": null }, { "pk": "${ZKAPP_SENDER_PUB_KEY}", "balance": "155.000000000", "delegate": null, "sk": null }, - { "pk": "${ZKAPP_ACCOUNT_PUB_KEY}", "balance": "155.000000000", "delegate": null, "sk": null }, + { "pk": "${ZKAPP_ACCOUNT_PUB_KEY}", "balance": "155.000000000", "delegate": null, "sk": null } ] } } From 16716c951c148be771f23187287690e8a09659b7 Mon Sep 17 00:00:00 2001 From: Serhii Shymkiv Date: Thu, 5 Oct 2023 13:27:32 +0300 Subject: [PATCH 15/32] Typo. --- buildkite/scripts/rosetta-integration-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/scripts/rosetta-integration-tests.sh b/buildkite/scripts/rosetta-integration-tests.sh index f6759f304f7..137f803f071 100755 --- a/buildkite/scripts/rosetta-integration-tests.sh +++ b/buildkite/scripts/rosetta-integration-tests.sh @@ -94,7 +94,7 @@ cat <"$MINA_CONFIG_FILE" "name": "${MINA_NETWORK}", "accounts": [ { "pk": "${BLOCK_PRODUCER_PK}", "balance": "11550000.000000000", "delegate": null, "sk": null }, - { "pk": "${SNARK_PRODUCER_PK}", "balance": "65500.000000000", "delegate": "${BLOCK_PRODUCER_PK}", "sk": null } + { "pk": "${SNARK_PRODUCER_PK}", "balance": "65500.000000000", "delegate": "${BLOCK_PRODUCER_PK}", "sk": null }, { "pk": "${ZKAPP_FEE_PAYER_PUB_KEY}", "balance": "155.000000000", "delegate": null, "sk": null }, { "pk": "${ZKAPP_SENDER_PUB_KEY}", "balance": "155.000000000", "delegate": null, "sk": null }, { "pk": "${ZKAPP_ACCOUNT_PUB_KEY}", "balance": "155.000000000", "delegate": null, "sk": null } From 47e96fddc8eda55b3188307a08835e443e911562 Mon Sep 17 00:00:00 2001 From: Serhii Shymkiv Date: Thu, 5 Oct 2023 15:09:43 +0300 Subject: [PATCH 16/32] Sending zkapp txns agaisnt Daemon's GraphQL endpoint. --- .../scripts/rosetta-integration-tests.sh | 41 ++++++++----------- 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/buildkite/scripts/rosetta-integration-tests.sh b/buildkite/scripts/rosetta-integration-tests.sh index 137f803f071..6450bd54342 100755 --- a/buildkite/scripts/rosetta-integration-tests.sh +++ b/buildkite/scripts/rosetta-integration-tests.sh @@ -179,54 +179,47 @@ until [ $daemon_status == "Synced" ]; do echo "Daemon Status: ${daemon_status}" done -debug_zkapp_output() { - echo "" - echo "" - echo "=========== DEBUG ZKAPP TXN (Start) ===========" - echo "" - echo "" - echo ${1} - echo "" - echo "" - echo "=========== DEBUG ZKAPP TXN (End) ===========" - echo "" - echo "" +send_zkapp_txn() { + local GRAPHQL_REQUEST="$1" + local ENDPOINT="http://127.0.0.1:${MINA_GRAPHQL_PORT}/graphql" + + curl -X POST \ + -H "Content-Type: application/json" \ + --data "{\"query\":\"$GRAPHQL_REQUEST\"}" \ + "$ENDPOINT" } echo "========================= ZKAPP ACCOUNT SETTING UP ===========================" -QUERY=$(zkapp_test_transaction create-zkapp-account --fee-payer-key ${ZKAPP_FEE_PAYER_KEY} --nonce 0 --sender-key ${ZKAPP_SENDER_KEY} --sender-nonce 0 --receiver-amount 1000 --zkapp-account-key ${ZKAPP_ACCOUNT_KEY} --fee 5 | sed 1,7d) -# TODO: Send txn against "http://127.0.0.1:${MINA_GRAPHQL_PORT}/graphql" -debug_zkapp_output "${QUERY}" +ZKAPP_TXN_QUERY=$(zkapp_test_transaction create-zkapp-account --fee-payer-key ${ZKAPP_FEE_PAYER_KEY} --nonce 0 --sender-key ${ZKAPP_SENDER_KEY} --sender-nonce 0 --receiver-amount 1000 --zkapp-account-key ${ZKAPP_ACCOUNT_KEY} --fee 5 | sed 1,7d) +send_zkapp_txn "${ZKAPP_TXN_QUERY}" # Unlock Genesis Accounts echo "==================== UNLOCKING GENESIS ACCOUNTS ======================" mina-dev accounts unlock --public-key $BLOCK_PRODUCER_PK mina-dev accounts unlock --public-key $SNARK_PRODUCER_PK -# Start sending payments -send_payments() { +# Start sending value transfer transactions +send_value_transfer_txns() { mina-dev client send-payment -rest-server http://127.0.0.1:${MINA_GRAPHQL_PORT}/graphql -amount 1 -nonce 0 -receiver $BLOCK_PRODUCER_PK -sender $BLOCK_PRODUCER_PK while true; do sleep $TRANSACTION_FREQUENCY mina-dev client send-payment -rest-server http://127.0.0.1:${MINA_GRAPHQL_PORT}/graphql -amount 1 -receiver $BLOCK_PRODUCER_PK -sender $BLOCK_PRODUCER_PK done } -send_payments & +send_value_transfer_txns & ZKAPP_FEE_PAYER_NONCE=1 ZKAPP_SENDER_NONCE=1 ZKAPP_STATE=0 send_zkapp_transactions() { while true; do - QUERY=$(zkapp_test_transaction transfer-funds-one-receiver --fee-payer-key ${ZKAPP_FEE_PAYER_KEY} --nonce ${ZKAPP_FEE_PAYER_NONCE} --sender-key ${ZKAPP_SENDER_KEY} --sender-nonce ${ZKAPP_SENDER_NONCE} --receiver-amount 1 --fee 5 --receiver ${ZKAPP_ACCOUNT_PUB_KEY} | sed 1,5d) - # TODO: Send txn against "http://127.0.0.1:${MINA_GRAPHQL_PORT}/graphql" - debug_zkapp_output "${QUERY}" + ZKAPP_TXN_QUERY=$(zkapp_test_transaction transfer-funds-one-receiver --fee-payer-key ${ZKAPP_FEE_PAYER_KEY} --nonce ${ZKAPP_FEE_PAYER_NONCE} --sender-key ${ZKAPP_SENDER_KEY} --sender-nonce ${ZKAPP_SENDER_NONCE} --receiver-amount 1 --fee 5 --receiver ${ZKAPP_ACCOUNT_PUB_KEY} | sed 1,5d) + send_zkapp_txn "${ZKAPP_TXN_QUERY}" let ZKAPP_FEE_PAYER_NONCE++ let ZKAPP_SENDER_NONCE++ - QUERY=$(zkapp_test_transaction update-state --fee-payer-key ${ZKAPP_FEE_PAYER_KEY} --nonce ${ZKAPP_FEE_PAYER_NONCE} --zkapp-account-key ${ZKAPP_SENDER_KEY} --zkapp-state ${ZKAPP_STATE} --fee 5 | sed 1,5d) - # TODO: Send txn against "http://127.0.0.1:${MINA_GRAPHQL_PORT}/graphql" - debug_zkapp_output "${QUERY}" + ZKAPP_TXN_QUERY=$(zkapp_test_transaction update-state --fee-payer-key ${ZKAPP_FEE_PAYER_KEY} --nonce ${ZKAPP_FEE_PAYER_NONCE} --zkapp-account-key ${ZKAPP_SENDER_KEY} --zkapp-state ${ZKAPP_STATE} --fee 5 | sed 1,5d) + send_zkapp_txn "${ZKAPP_TXN_QUERY}" let ZKAPP_FEE_PAYER_NONCE++ let ZKAPP_STATE++ done From f07fa96c176a453f0ab074d48c2049b081d8e504 Mon Sep 17 00:00:00 2001 From: Serhii Shymkiv Date: Thu, 5 Oct 2023 16:36:14 +0300 Subject: [PATCH 17/32] Let's try with Python. --- .../scripts/rosetta-integration-tests.sh | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/buildkite/scripts/rosetta-integration-tests.sh b/buildkite/scripts/rosetta-integration-tests.sh index 6450bd54342..7f8757e8580 100755 --- a/buildkite/scripts/rosetta-integration-tests.sh +++ b/buildkite/scripts/rosetta-integration-tests.sh @@ -163,7 +163,6 @@ mina-dev daemon \ --libp2p-keypair ${MINA_LIBP2P_KEYPAIR_PATH} \ --log-level ${LOG_LEVEL} \ --proof-level none \ - --insecure-rest-server \ --rest-port ${MINA_GRAPHQL_PORT} \ --run-snark-worker "$SNARK_PRODUCER_PK" \ --seed \ @@ -180,13 +179,16 @@ until [ $daemon_status == "Synced" ]; do done send_zkapp_txn() { - local GRAPHQL_REQUEST="$1" - local ENDPOINT="http://127.0.0.1:${MINA_GRAPHQL_PORT}/graphql" - - curl -X POST \ - -H "Content-Type: application/json" \ - --data "{\"query\":\"$GRAPHQL_REQUEST\"}" \ - "$ENDPOINT" + local url="http://127.0.0.1:${MINA_GRAPHQL_PORT}/graphql" + local query="$1" + + python3 < Date: Thu, 5 Oct 2023 17:47:07 +0300 Subject: [PATCH 18/32] python3 -> python --- buildkite/scripts/rosetta-integration-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/scripts/rosetta-integration-tests.sh b/buildkite/scripts/rosetta-integration-tests.sh index 7f8757e8580..9556c5befce 100755 --- a/buildkite/scripts/rosetta-integration-tests.sh +++ b/buildkite/scripts/rosetta-integration-tests.sh @@ -182,7 +182,7 @@ send_zkapp_txn() { local url="http://127.0.0.1:${MINA_GRAPHQL_PORT}/graphql" local query="$1" - python3 < Date: Thu, 5 Oct 2023 20:10:02 +0300 Subject: [PATCH 19/32] Install python. --- .../scripts/rosetta-integration-tests.sh | 6 +- dockerfiles/stages/1-build-deps | 63 ++++++++++--------- 2 files changed, 37 insertions(+), 32 deletions(-) diff --git a/buildkite/scripts/rosetta-integration-tests.sh b/buildkite/scripts/rosetta-integration-tests.sh index 9556c5befce..c50806c3ce9 100755 --- a/buildkite/scripts/rosetta-integration-tests.sh +++ b/buildkite/scripts/rosetta-integration-tests.sh @@ -178,11 +178,15 @@ until [ $daemon_status == "Synced" ]; do echo "Daemon Status: ${daemon_status}" done +echo "--- Which Python? ---" +which python +which python3 + send_zkapp_txn() { local url="http://127.0.0.1:${MINA_GRAPHQL_PORT}/graphql" local query="$1" - python </dev/null \ - || make -C /rocksdb static_lib PORTABLE=1 -j$(nproc) 2>/dev/null \ + && CXXFLAGS='' make -C /rocksdb static_lib PORTABLE=1 -j$(nproc) 2>/dev/null \ + || make -C /rocksdb static_lib PORTABLE=1 -j$(nproc) 2>/dev/null \ && cp /rocksdb/librocksdb.a /usr/local/lib/librocksdb_coda.a \ && rm -rf /rocksdb \ && strip -S /usr/local/lib/librocksdb_coda.a From 1ba568993609b22d425b76793d30456cfc070539 Mon Sep 17 00:00:00 2001 From: Serhii Shymkiv Date: Thu, 5 Oct 2023 21:20:03 +0300 Subject: [PATCH 20/32] Install python. --- buildkite/scripts/rosetta-integration-tests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/buildkite/scripts/rosetta-integration-tests.sh b/buildkite/scripts/rosetta-integration-tests.sh index c50806c3ce9..e2c77476271 100755 --- a/buildkite/scripts/rosetta-integration-tests.sh +++ b/buildkite/scripts/rosetta-integration-tests.sh @@ -179,8 +179,8 @@ until [ $daemon_status == "Synced" ]; do done echo "--- Which Python? ---" -which python -which python3 +which python || true +which python3 || true send_zkapp_txn() { local url="http://127.0.0.1:${MINA_GRAPHQL_PORT}/graphql" From 645a76891df7b4324d05afb77ecf1f3368ba691d Mon Sep 17 00:00:00 2001 From: Serhii Shymkiv Date: Thu, 5 Oct 2023 22:24:58 +0300 Subject: [PATCH 21/32] Install python and deps when needed (debugging attempt). --- buildkite/scripts/rosetta-integration-tests.sh | 6 +++++- dockerfiles/stages/1-build-deps | 1 - 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/buildkite/scripts/rosetta-integration-tests.sh b/buildkite/scripts/rosetta-integration-tests.sh index e2c77476271..8e7334d9cdb 100755 --- a/buildkite/scripts/rosetta-integration-tests.sh +++ b/buildkite/scripts/rosetta-integration-tests.sh @@ -178,7 +178,11 @@ until [ $daemon_status == "Synced" ]; do echo "Daemon Status: ${daemon_status}" done -echo "--- Which Python? ---" +# Install Python3 and dependencies +apt-get install --no-install-recommends --quiet --yes python3 python3-pip +pip3 install requests + +echo "Which Python?" which python || true which python3 || true diff --git a/dockerfiles/stages/1-build-deps b/dockerfiles/stages/1-build-deps index 25411928851..81d53f72a40 100644 --- a/dockerfiles/stages/1-build-deps +++ b/dockerfiles/stages/1-build-deps @@ -64,7 +64,6 @@ RUN apt-get update --quiet \ sudo \ unzip \ zlib1g-dev \ - python3 \ && rm -rf /var/lib/apt/lists/* # Symlink image-specific lld version to a single lld executable From b6a842a11eb899489a5793be1f9c1f835dfd084c Mon Sep 17 00:00:00 2001 From: Serhii Shymkiv Date: Fri, 6 Oct 2023 10:01:45 +0300 Subject: [PATCH 22/32] Escaping quotes with jq. --- .../scripts/rosetta-integration-tests.sh | 26 +++++++------------ 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/buildkite/scripts/rosetta-integration-tests.sh b/buildkite/scripts/rosetta-integration-tests.sh index 8e7334d9cdb..36bb3f17da2 100755 --- a/buildkite/scripts/rosetta-integration-tests.sh +++ b/buildkite/scripts/rosetta-integration-tests.sh @@ -178,25 +178,17 @@ until [ $daemon_status == "Synced" ]; do echo "Daemon Status: ${daemon_status}" done -# Install Python3 and dependencies -apt-get install --no-install-recommends --quiet --yes python3 python3-pip -pip3 install requests - -echo "Which Python?" -which python || true -which python3 || true +# Install jq +apt-get install --no-install-recommends --quiet --yes jq send_zkapp_txn() { - local url="http://127.0.0.1:${MINA_GRAPHQL_PORT}/graphql" - local query="$1" - - python3 < Date: Fri, 6 Oct 2023 13:10:23 +0300 Subject: [PATCH 23/32] Tested network comms locally. --- .../scripts/rosetta-integration-tests.sh | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/buildkite/scripts/rosetta-integration-tests.sh b/buildkite/scripts/rosetta-integration-tests.sh index 36bb3f17da2..a1a3f25f051 100755 --- a/buildkite/scripts/rosetta-integration-tests.sh +++ b/buildkite/scripts/rosetta-integration-tests.sh @@ -1,11 +1,9 @@ #!/bin/bash +set -eox pipefail # These tests use the mina-dev binary, as rosetta-cli assumes we use a testnet. # See https://github.com/coinbase/rosetta-sdk-go/blob/master/keys/signer_pallas.go#L222 -set -eo pipefail -#! /bin/bash - # Defines scope of test. Currently supported are: # - minimal -> only quick checks (~5 mins) # - full -> all checks @@ -93,11 +91,11 @@ cat <"$MINA_CONFIG_FILE" "ledger": { "name": "${MINA_NETWORK}", "accounts": [ - { "pk": "${BLOCK_PRODUCER_PK}", "balance": "11550000.000000000", "delegate": null, "sk": null }, - { "pk": "${SNARK_PRODUCER_PK}", "balance": "65500.000000000", "delegate": "${BLOCK_PRODUCER_PK}", "sk": null }, - { "pk": "${ZKAPP_FEE_PAYER_PUB_KEY}", "balance": "155.000000000", "delegate": null, "sk": null }, - { "pk": "${ZKAPP_SENDER_PUB_KEY}", "balance": "155.000000000", "delegate": null, "sk": null }, - { "pk": "${ZKAPP_ACCOUNT_PUB_KEY}", "balance": "155.000000000", "delegate": null, "sk": null } + { "pk": "${BLOCK_PRODUCER_PK}", "balance": "101550000.000000000", "delegate": null, "sk": null }, + { "pk": "${SNARK_PRODUCER_PK}", "balance": "605500.000000000", "delegate": "${BLOCK_PRODUCER_PK}", "sk": null }, + { "pk": "${ZKAPP_FEE_PAYER_PUB_KEY}", "balance": "10055.000000000", "delegate": null, "sk": null }, + { "pk": "${ZKAPP_SENDER_PUB_KEY}", "balance": "10055.000000000", "delegate": null, "sk": null }, + { "pk": "${ZKAPP_ACCOUNT_PUB_KEY}", "balance": "10055.000000000", "delegate": null, "sk": null } ] } } @@ -178,16 +176,14 @@ until [ $daemon_status == "Synced" ]; do echo "Daemon Status: ${daemon_status}" done -# Install jq -apt-get install --no-install-recommends --quiet --yes jq - send_zkapp_txn() { local GRAPHQL_REQUEST="$1" + local ESCAPED_GRAPHQL_REQUEST="${GRAPHQL_REQUEST//\"/\\\"}" local ENDPOINT="http://127.0.0.1:${MINA_GRAPHQL_PORT}/graphql" curl -X POST \ -H "Content-Type: application/json" \ - --data "$(echo "{\"query\": $(echo "$GRAPHQL_REQUEST" | jq -R .)}")" \ + --data "{\"query\":\"$ESCAPED_GRAPHQL_REQUEST\"}" \ "$ENDPOINT" } From 535358ff0497de3bb26d620b70e436b85c035a77 Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 9 Oct 2023 23:56:22 +0200 Subject: [PATCH 24/32] test full in PR --- buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall index 36106915503..be26298998a 100644 --- a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall +++ b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall @@ -40,7 +40,7 @@ Pipeline.build Command.Config::{ commands = [ Cmd.run ("export MINA_DEB_CODENAME=bullseye && source ./buildkite/scripts/export-git-env-vars.sh && echo \\\${MINA_DOCKER_TAG}"), - Cmd.runInDocker Cmd.Docker::{image="gcr.io/o1labs-192920/mina-rosetta:\\\${MINA_DOCKER_TAG}", entrypoint=" --entrypoint buildkite/scripts/rosetta-integration-tests-fast.sh"} "bash" + Cmd.runInDocker Cmd.Docker::{image="gcr.io/o1labs-192920/mina-rosetta:\\\${MINA_DOCKER_TAG}", entrypoint=" --entrypoint buildkite/scripts/rosetta-integration-tests-full.sh"} "bash" ], label = "Rosetta integration tests Bullseye" , key = "rosetta-integration-tests-bullseye" From 0ebb8a691deafc31be016ab69f1f784adc2999ea Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 10 Oct 2023 09:43:06 +0200 Subject: [PATCH 25/32] log statements in rosetta --- src/app/rosetta/postgresql.conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/app/rosetta/postgresql.conf b/src/app/rosetta/postgresql.conf index 30736eb3be8..965634234d6 100644 --- a/src/app/rosetta/postgresql.conf +++ b/src/app/rosetta/postgresql.conf @@ -3,3 +3,6 @@ max_locks_per_transaction=100 max_pred_locks_per_relation=100 max_pred_locks_per_transaction=5000 max_connections=500 +log_statement='all' +logging_collector='on' +log_directory = 'pg_log' \ No newline at end of file From 0ee21c1c8356da15599250314e38ae80dbbc088b Mon Sep 17 00:00:00 2001 From: Sventimir Date: Wed, 18 Oct 2023 10:22:39 +0200 Subject: [PATCH 26/32] [rosetta] Fix broken SQL quesry for zkapp account updates. --- src/app/rosetta/lib/block.ml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/app/rosetta/lib/block.ml b/src/app/rosetta/lib/block.ml index 6d9d22329c9..1db0a0f7b4d 100644 --- a/src/app/rosetta/lib/block.ml +++ b/src/app/rosetta/lib/block.ml @@ -998,9 +998,12 @@ module Sql = struct zaub.call_depth, zaub.zkapp_network_precondition_id, zaub.zkapp_account_precondition_id, + zaub.zkapp_valid_while_precondition_id, zaub.use_full_commitment, + zaub.implicit_account_creation_fee, zaub.may_use_token, zaub.authorization_kind, + zaub.verification_key_hash_id, pk.value as account, bzc.status FROM zkapp_commands zc From a54d385b2feaf65d82ba03fd095565cdfe5d9e3b Mon Sep 17 00:00:00 2001 From: dkijania Date: Wed, 18 Oct 2023 16:07:35 +0200 Subject: [PATCH 27/32] lint mode parameter and exclude sending transactions from minimal mode --- buildkite/scripts/rosetta-integration-tests.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/buildkite/scripts/rosetta-integration-tests.sh b/buildkite/scripts/rosetta-integration-tests.sh index a1a3f25f051..594aff60f39 100755 --- a/buildkite/scripts/rosetta-integration-tests.sh +++ b/buildkite/scripts/rosetta-integration-tests.sh @@ -223,7 +223,11 @@ send_zkapp_transactions() { let ZKAPP_STATE++ done } -send_zkapp_transactions & + +# There is no point to generate transaction just for minimal mode +if [[ "$MODE" == "full" ]]; then + send_zkapp_transactions & +fi next_block_time=$(mina-dev client status --json | jq '.next_block_production.timing[1].time' | tr -d '"') curr_time=$(date +%s%N | cut -b1-13) sleep_time=$((($next_block_time - $curr_time) / 1000)) @@ -237,7 +241,7 @@ rosetta-cli configuration:validate ${ROSETTA_CONFIGURATION_FILE} echo "========================= ROSETTA CLI: CHECK:SPEC ===========================" rosetta-cli check:spec --all --configuration-file ${ROSETTA_CONFIGURATION_FILE} -if [[ $MODE == "full" ]]; then +if [[ "$MODE" == "full" ]]; then echo "========================= ROSETTA CLI: CHECK:CONSTRUCTION ===========================" rosetta-cli check:construction --configuration-file ${ROSETTA_CONFIGURATION_FILE} From 9c109045702c6c370a7b01096d2da93dcd73fecf Mon Sep 17 00:00:00 2001 From: dkijania Date: Thu, 19 Oct 2023 21:12:54 +0200 Subject: [PATCH 28/32] add zkapp test transaction --- dockerfiles/Dockerfile-mina-rosetta | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerfiles/Dockerfile-mina-rosetta b/dockerfiles/Dockerfile-mina-rosetta index 7b3e8cc78df..b3ccc4b3235 100755 --- a/dockerfiles/Dockerfile-mina-rosetta +++ b/dockerfiles/Dockerfile-mina-rosetta @@ -96,7 +96,7 @@ RUN mkdir -p --mode=700 ${MINA_CONFIG_DIR}/wallets/store/ \ RUN echo "Building image with version $deb_version from repo $deb_release $deb_codename for network $network" \ && echo "deb [trusted=yes] http://packages.o1test.net $deb_codename $deb_release" > /etc/apt/sources.list.d/o1.list \ && apt-get update --quiet --yes \ - && apt-get install --quiet --yes --allow-downgrades "mina-$network=$deb_version" "mina-archive=$deb_version" \ + && apt-get install --quiet --yes --allow-downgrades "mina-$network=$deb_version" "mina-archive=$deb_version" "zkapp_test_transaction=$deb_version" \ && rm -rf /var/lib/apt/lists/* # --- Set up postgres From b45f07aa214267938635170e9c796ae4a3cb5a94 Mon Sep 17 00:00:00 2001 From: dkijania Date: Thu, 19 Oct 2023 21:39:03 +0200 Subject: [PATCH 29/32] use correct deb --- buildkite/scripts/rosetta-integration-tests.sh | 6 +++--- dockerfiles/Dockerfile-mina-rosetta | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/buildkite/scripts/rosetta-integration-tests.sh b/buildkite/scripts/rosetta-integration-tests.sh index aaf976a92e8..2878fe03198 100755 --- a/buildkite/scripts/rosetta-integration-tests.sh +++ b/buildkite/scripts/rosetta-integration-tests.sh @@ -185,7 +185,7 @@ send_zkapp_txn() { } echo "========================= ZKAPP ACCOUNT SETTING UP ===========================" -ZKAPP_TXN_QUERY=$(zkapp_test_transaction create-zkapp-account --fee-payer-key ${ZKAPP_FEE_PAYER_KEY} --nonce 0 --sender-key ${ZKAPP_SENDER_KEY} --sender-nonce 0 --receiver-amount 1000 --zkapp-account-key ${ZKAPP_ACCOUNT_KEY} --fee 5 | sed 1,7d) +ZKAPP_TXN_QUERY=$(mina-zkapp-test-transaction create-zkapp-account --fee-payer-key ${ZKAPP_FEE_PAYER_KEY} --nonce 0 --sender-key ${ZKAPP_SENDER_KEY} --sender-nonce 0 --receiver-amount 1000 --zkapp-account-key ${ZKAPP_ACCOUNT_KEY} --fee 5 | sed 1,7d) send_zkapp_txn "${ZKAPP_TXN_QUERY}" # Unlock Genesis Accounts @@ -210,12 +210,12 @@ ZKAPP_SENDER_NONCE=1 ZKAPP_STATE=0 send_zkapp_transactions() { while true; do - ZKAPP_TXN_QUERY=$(zkapp_test_transaction transfer-funds-one-receiver --fee-payer-key ${ZKAPP_FEE_PAYER_KEY} --nonce ${ZKAPP_FEE_PAYER_NONCE} --sender-key ${ZKAPP_SENDER_KEY} --sender-nonce ${ZKAPP_SENDER_NONCE} --receiver-amount 1 --fee 5 --receiver ${ZKAPP_ACCOUNT_PUB_KEY} | sed 1,5d) + ZKAPP_TXN_QUERY=$(mina-zkapp-test-transaction transfer-funds-one-receiver --fee-payer-key ${ZKAPP_FEE_PAYER_KEY} --nonce ${ZKAPP_FEE_PAYER_NONCE} --sender-key ${ZKAPP_SENDER_KEY} --sender-nonce ${ZKAPP_SENDER_NONCE} --receiver-amount 1 --fee 5 --receiver ${ZKAPP_ACCOUNT_PUB_KEY} | sed 1,5d) send_zkapp_txn "${ZKAPP_TXN_QUERY}" let ZKAPP_FEE_PAYER_NONCE++ let ZKAPP_SENDER_NONCE++ - ZKAPP_TXN_QUERY=$(zkapp_test_transaction update-state --fee-payer-key ${ZKAPP_FEE_PAYER_KEY} --nonce ${ZKAPP_FEE_PAYER_NONCE} --zkapp-account-key ${ZKAPP_SENDER_KEY} --zkapp-state ${ZKAPP_STATE} --fee 5 | sed 1,5d) + ZKAPP_TXN_QUERY=$(mina-zkapp-test-transaction update-state --fee-payer-key ${ZKAPP_FEE_PAYER_KEY} --nonce ${ZKAPP_FEE_PAYER_NONCE} --zkapp-account-key ${ZKAPP_SENDER_KEY} --zkapp-state ${ZKAPP_STATE} --fee 5 | sed 1,5d) send_zkapp_txn "${ZKAPP_TXN_QUERY}" let ZKAPP_FEE_PAYER_NONCE++ let ZKAPP_STATE++ diff --git a/dockerfiles/Dockerfile-mina-rosetta b/dockerfiles/Dockerfile-mina-rosetta index b3ccc4b3235..525778a1d30 100755 --- a/dockerfiles/Dockerfile-mina-rosetta +++ b/dockerfiles/Dockerfile-mina-rosetta @@ -96,7 +96,7 @@ RUN mkdir -p --mode=700 ${MINA_CONFIG_DIR}/wallets/store/ \ RUN echo "Building image with version $deb_version from repo $deb_release $deb_codename for network $network" \ && echo "deb [trusted=yes] http://packages.o1test.net $deb_codename $deb_release" > /etc/apt/sources.list.d/o1.list \ && apt-get update --quiet --yes \ - && apt-get install --quiet --yes --allow-downgrades "mina-$network=$deb_version" "mina-archive=$deb_version" "zkapp_test_transaction=$deb_version" \ + && apt-get install --quiet --yes --allow-downgrades "mina-$network=$deb_version" "mina-archive=$deb_version" "mina-zkapp-test-transaction=$deb_version" \ && rm -rf /var/lib/apt/lists/* # --- Set up postgres From c9c3bb8566c7bfdaf51eb2fcc9f1ad950e5b2533 Mon Sep 17 00:00:00 2001 From: dkijania Date: Thu, 19 Oct 2023 22:34:09 +0200 Subject: [PATCH 30/32] add missing keys --- buildkite/scripts/rosetta-integration-tests.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/buildkite/scripts/rosetta-integration-tests.sh b/buildkite/scripts/rosetta-integration-tests.sh index 2878fe03198..a93d7c2bc66 100755 --- a/buildkite/scripts/rosetta-integration-tests.sh +++ b/buildkite/scripts/rosetta-integration-tests.sh @@ -92,7 +92,10 @@ cat <"$MINA_CONFIG_FILE" "name": "${MINA_NETWORK}", "accounts": [ { "pk": "${BLOCK_PRODUCER_PK}", "balance": "1000000", "delegate": null, "sk": null }, - { "pk": "${SNARK_PRODUCER_PK}", "balance": "2000000", "delegate": "${BLOCK_PRODUCER_PK}", "sk": null } + { "pk": "${SNARK_PRODUCER_PK}", "balance": "2000000", "delegate": "${BLOCK_PRODUCER_PK}", "sk": null }, + { "pk": "${ZKAPP_FEE_PAYER_PUB_KEY}", "balance": "1000000", "delegate": null, "sk": null }, + { "pk": "${ZKAPP_SENDER_PUB_KEY}", "balance": "1000000", "delegate": null, "sk": null }, + { "pk": "${ZKAPP_ACCOUNT_PUB_KEY}", "balance": "1000000", "delegate": null, "sk": null } ] } } @@ -196,7 +199,7 @@ mina accounts unlock --public-key $SNARK_PRODUCER_PK # Start sending value transfer transactions send_value_transfer_txns() { - mina-dev client send-payment -rest-server http://127.0.0.1:${MINA_GRAPHQL_PORT}/graphql -amount 1 -nonce 0 -receiver $BLOCK_PRODUCER_PK -sender $BLOCK_PRODUCER_PK + mina client send-payment -rest-server http://127.0.0.1:${MINA_GRAPHQL_PORT}/graphql -amount 1 -nonce 0 -receiver $BLOCK_PRODUCER_PK -sender $BLOCK_PRODUCER_PK while true; do sleep $TRANSACTION_FREQUENCY mina client send-payment -rest-server http://127.0.0.1:${MINA_GRAPHQL_PORT}/graphql -amount 1 -receiver $BLOCK_PRODUCER_PK -sender $BLOCK_PRODUCER_PK @@ -227,7 +230,7 @@ if [[ "$MODE" == "full" ]]; then send_zkapp_transactions & fi -next_block_time=$(mina-dev client status --json | jq '.next_block_production.timing[1].time' | tr -d '"') curr_time=$(date +%s%N | cut -b1-13) +next_block_time=$(mina client status --json | jq '.next_block_production.timing[1].time' | tr -d '"') curr_time=$(date +%s%N | cut -b1-13) sleep_time=$((($next_block_time - $curr_time) / 1000)) echo "Sleeping for ${sleep_time}s until next block is created..." sleep ${sleep_time} From 0cc7951cd58456a2fcb6ce53d66874b64b159c78 Mon Sep 17 00:00:00 2001 From: dkijania Date: Fri, 20 Oct 2023 16:50:14 +0200 Subject: [PATCH 31/32] revert running fast checks on PR --- buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall index eb9b0a297b7..6daca1a254d 100644 --- a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall +++ b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall @@ -42,7 +42,7 @@ Pipeline.build Command.Config::{ commands = [ Cmd.run ("export MINA_DEB_CODENAME=bullseye && source ./buildkite/scripts/export-git-env-vars.sh && echo \\\${MINA_DOCKER_TAG}"), - Cmd.runInDocker Cmd.Docker::{image="gcr.io/o1labs-192920/mina-rosetta:\\\${MINA_DOCKER_TAG}", entrypoint=" --entrypoint buildkite/scripts/rosetta-integration-tests-full.sh"} "bash" + Cmd.runInDocker Cmd.Docker::{image="gcr.io/o1labs-192920/mina-rosetta:\\\${MINA_DOCKER_TAG}", entrypoint=" --entrypoint buildkite/scripts/rosetta-integration-tests-fast.sh"} "bash" ], label = "Rosetta integration tests Bullseye" , key = "rosetta-integration-tests-bullseye" From 026d53852c286178de7f4a532e725673b880e953 Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 23 Oct 2023 10:03:40 +0200 Subject: [PATCH 32/32] adjusting DirtyWhen --- buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall | 2 +- buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall index 6daca1a254d..3cdce9b6276 100644 --- a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall +++ b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall @@ -21,7 +21,7 @@ let dirtyWhen = [ S.strictlyStart (S.contains "src/app/archive"), S.exactly "buildkite/src/Jobs/Test/RosettaIntegrationTests" "dhall", S.exactly "buildkite/scripts/rosetta-integration-tests" "sh", - S.exactly "buildkite/scripts/rosetta-integration-tests-long.sh" "sh" + S.exactly "buildkite/scripts/rosetta-integration-tests-fast" "sh" ] let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type diff --git a/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall b/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall index c2af7b5be95..e836856cc4d 100644 --- a/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall +++ b/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall @@ -21,7 +21,7 @@ let dirtyWhen = [ S.strictlyStart (S.contains "src/app/archive"), S.exactly "buildkite/src/Jobs/Test/RosettaIntegrationTests" "dhall", S.exactly "buildkite/scripts/rosetta-integration-tests" "sh", - S.exactly "buildkite/scripts/rosetta-integration-tests-fast.sh" "sh" + S.exactly "buildkite/scripts/rosetta-integration-tests-full" "sh" ] let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type