Merge pull request #14224 from MinaProtocol/dkijania/tailor_rosetta_i…

…nt_tests

run only fast checks in rosetta int tests

buildkite/scripts/rosetta-integration-tests-fast.sh

@@ -0,0 +1 @@
+buildkite/scripts/rosetta-integration-tests.sh --mode=minimal

buildkite/scripts/rosetta-integration-tests-full.sh

@@ -0,0 +1 @@
+buildkite/scripts/rosetta-integration-tests.sh --mode=full

buildkite/scripts/rosetta-integration-tests.sh

@@ -1,21 +1,22 @@
 #!/bin/bash
-
-# Deploy a sandboxed Mina daemon with an archive and a Rosetta instance.
-# Deploy 2 zkApps from https://github.com/MinaProtocol/rosetta-integration-test-zkapps
-# to the network, interact with them and add some regular transactions.
-# Then run full rosetta-cli tests against the Rosetta instance.
-
-# NPM and NodeJS are installed through NVM, versions are stored in environment
-# variables below. Zkapp-cli is installed globally through NPM, however, to
-# ensure compatibility with the daemon, we use o1js pinned in the Mina repo.
-# The repo is mounted into the container at /workdir, so we can build o1js from
-# that source. It is important to make sure that the zkapp-cli version installed
-# is compatible with o1js version used.
+set -eox pipefail
 
 # These tests use the mina binary, as rosetta-cli assumes we use a testnet.
 # See https://github.com/coinbase/rosetta-sdk-go/blob/master/keys/signer_pallas.go#L222
 
-set -eo pipefail
+# Defines scope of test. Currently supported are:
+# - minimal -> only quick checks (~5 mins)
+# - full -> all checks
+MODE="minimal"
+
+while [ $# -gt 0 ]; do
+  case "$1" in
+  --mode=*)
+    MODE="${1#*=}"
+    ;;
+  esac
+  shift
+done
 
 export MINA_NETWORK=${MINA_NETWORK:=sandbox}
 export LOG_LEVEL="${LOG_LEVEL:=Info}"
@@ -47,43 +48,6 @@ export MINA_CONFIG_FILE=$HOME/${MINA_NETWORK}.json
 export MINA_CONFIG_DIR="${MINA_CONFIG_DIR:=$HOME/.mina-config}"
 export MINA_GRAPHQL_PORT=${MINA_GRAPHQL_PORT:=3085}
 
-# Test variables
-export ROSETTA_INT_TEST_ZKAPPS_VERSION=${ROSETTA_INT_TEST_ZKAPPS_VERSION:=rosetta-ci-tests}
-
-# We need a version which is compatible with o1js pinned to the Mina repo.
-# Should be set to 'latest' most of the time, but occasionally we might need
-# an older one.
-export ZKAPP_CLI_VERSION=0.11.0
-
-# Nodejs variables
-export NVM_VERSION=0.39.3
-export NODE_VERSION=20.6.1
-
-# zkApps variables
-export ZKAPP_PATH=$HOME/zkapps
-
-echo "=========================== INSTALLING NPM ==========================="
-curl -so- https://raw.githubusercontent.com/nvm-sh/nvm/v${NVM_VERSION}/install.sh | bash &>/dev/null
-export NVM_DIR="$HOME/.nvm"
-[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"                   # This loads nvm
-[ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion" # This loads nvm bash_completion
-
-nvm install $NODE_VERSION
-nvm use --delete-prefix $NODE_VERSION
-
-mkdir ~/.npm-global
-npm config set prefix '~/.npm-global'
-export PATH=~/.npm-global/bin:$PATH
-
-# Install zkapp-cli and Typescript compiler.
-npm install --no-progress --global "zkapp-cli@${ZKAPP_CLI_VERSION}" "typescript@latest"
-
-# Build o1js so that we can use it later.
-pushd /workdir/src/lib/snarkyjs
-npm ci
-npm run build
-popd
-
 # Rosetta CLI variables
 # Files from ROSETTA_CLI_CONFIG_FILES will be read from
 # ROSETTA_CONFIGURATION_INPUT_DIR and some placeholders will be
@@ -95,10 +59,6 @@ ROSETTA_CLI_MAIN_CONFIG_FILE=${ROSETTA_CLI_MAIN_CONFIG_FILE:="config.json"}
 # Frequency (in seconds) at which payment operations will be sent
 TRANSACTION_FREQUENCY=10
 
-# Fetch zkApps
-curl -Ls https://github.com/MinaProtocol/rosetta-integration-test-zkapps/tarball/$ROSETTA_INT_TEST_ZKAPPS_VERSION | tar xz -C /tmp
-mv /tmp/MinaProtocol-rosetta-integration-test-zkapps-* $ZKAPP_PATH
-
 # Libp2p Keypair
 echo "=========================== GENERATING KEYPAIR IN ${MINA_LIBP2P_KEYPAIR_PATH} ==========================="
 mina libp2p generate-keypair -privkey-path $MINA_LIBP2P_KEYPAIR_PATH
@@ -108,9 +68,18 @@ echo "=========================== GENERATING GENESIS LEDGER FOR ${MINA_NETWORK}
 mkdir -p $MINA_KEYS_PATH
 mina advanced generate-keypair --privkey-path $MINA_KEYS_PATH/block-producer.key
 mina advanced generate-keypair --privkey-path $MINA_KEYS_PATH/snark-producer.key
+mina advanced generate-keypair --privkey-path $MINA_KEYS_PATH/zkapp-fee-payer.key
+mina advanced generate-keypair --privkey-path $MINA_KEYS_PATH/zkapp-sender.key
+mina advanced generate-keypair --privkey-path $MINA_KEYS_PATH/zkapp-account.key
 chmod -R 0700 $MINA_KEYS_PATH
 BLOCK_PRODUCER_PK=$(cat $MINA_KEYS_PATH/block-producer.key.pub)
 SNARK_PRODUCER_PK=$(cat $MINA_KEYS_PATH/snark-producer.key.pub)
+ZKAPP_FEE_PAYER_KEY=$MINA_KEYS_PATH/zkapp-fee-payer.key
+ZKAPP_FEE_PAYER_PUB_KEY=$(cat ${ZKAPP_FEE_PAYER_KEY}.pub)
+ZKAPP_SENDER_KEY=$MINA_KEYS_PATH/zkapp-sender.key
+ZKAPP_SENDER_PUB_KEY=$(cat ${ZKAPP_SENDER_KEY}.pub)
+ZKAPP_ACCOUNT_KEY=$MINA_KEYS_PATH/zkapp-account.key
+ZKAPP_ACCOUNT_PUB_KEY=$(cat ${ZKAPP_ACCOUNT_KEY}.pub)
 
 mkdir -p $MINA_CONFIG_DIR/wallets/store
 cp $MINA_KEYS_PATH/block-producer.key $MINA_CONFIG_DIR/wallets/store/$BLOCK_PRODUCER_PK
@@ -123,24 +92,14 @@ cat <<EOF >"$MINA_CONFIG_FILE"
     "name": "${MINA_NETWORK}",
     "accounts": [
       { "pk": "${BLOCK_PRODUCER_PK}", "balance": "1000000", "delegate": null, "sk": null },
-      { "pk": "${SNARK_PRODUCER_PK}", "balance": "2000000", "delegate": "${BLOCK_PRODUCER_PK}", "sk": null }
+      { "pk": "${SNARK_PRODUCER_PK}", "balance": "2000000", "delegate": "${BLOCK_PRODUCER_PK}", "sk": null },
+      { "pk": "${ZKAPP_FEE_PAYER_PUB_KEY}", "balance": "1000000", "delegate": null, "sk": null },
+      { "pk": "${ZKAPP_SENDER_PUB_KEY}", "balance": "1000000", "delegate": null, "sk": null },
+      { "pk": "${ZKAPP_ACCOUNT_PUB_KEY}", "balance": "1000000", "delegate": null, "sk": null }
     ]
   }
 }
 EOF
-for zkapp_path in ${ZKAPP_PATH}/*/; do
-  zkapp_path=${zkapp_path%/}
-  zkapp=$(basename $zkapp_path)
-  # Generate zkApp account keypair
-  mina advanced generate-keypair --privkey-path ${MINA_KEYS_PATH}/zkapp-${zkapp}-account.key
-  # Generate zkApp fee payer keypair
-  mina advanced generate-keypair --privkey-path ${MINA_KEYS_PATH}/zkapp-${zkapp}-fee-payer.key
-  zkapp_fee_payer_pk=$(cat $MINA_KEYS_PATH/zkapp-${zkapp}-fee-payer.key.pub)
-  line="[{ \"pk\": \"${zkapp_fee_payer_pk}\", \"balance\": \"10000\", \"delegate\": null, \"sk\": null }]"
-  jq ".ledger.accounts |= . + ${line}" $MINA_CONFIG_FILE >${MINA_CONFIG_FILE}.tmp
-  mv ${MINA_CONFIG_FILE}.tmp $MINA_CONFIG_FILE
-done
-cat $MINA_CONFIG_FILE | jq .
 
 # Substitute placeholders in rosetta-cli configuration
 ROSETTA_CONFIGURATION_OUTPUT_DIR=/tmp/rosetta-cli-config
@@ -160,11 +119,6 @@ done
 echo "==================== IMPORTING GENESIS ACCOUNTS ======================"
 mina accounts import --privkey-path $MINA_KEYS_PATH/block-producer.key --config-directory $MINA_CONFIG_DIR
 mina accounts import --privkey-path $MINA_KEYS_PATH/snark-producer.key --config-directory $MINA_CONFIG_DIR
-for zkapp_path in ${ZKAPP_PATH}/*/; do
-  zkapp_path=${zkapp_path%/}
-  zkapp=$(basename $zkapp_path)
-  mina accounts import --privkey-path $MINA_KEYS_PATH/zkapp-${zkapp}-fee-payer.key --config-directory $MINA_CONFIG_DIR
-done
 
 # Postgres
 echo "========================= INITIALIZING POSTGRESQL ==========================="
@@ -222,100 +176,61 @@ until [ $daemon_status == "Synced" ]; do
   echo "Daemon Status: ${daemon_status}"
 done
 
+send_zkapp_txn() {
+  local GRAPHQL_REQUEST="$1"
+  local ESCAPED_GRAPHQL_REQUEST="${GRAPHQL_REQUEST//\"/\\\"}"
+  local ENDPOINT="http://127.0.0.1:${MINA_GRAPHQL_PORT}/graphql"
+
+  curl -X POST \
+    -H "Content-Type: application/json" \
+    --data "{\"query\":\"$ESCAPED_GRAPHQL_REQUEST\"}" \
+    "$ENDPOINT"
+}
+
+echo "========================= ZKAPP ACCOUNT SETTING UP ==========================="
+ZKAPP_TXN_QUERY=$(mina-zkapp-test-transaction create-zkapp-account --fee-payer-key ${ZKAPP_FEE_PAYER_KEY} --nonce 0 --sender-key ${ZKAPP_SENDER_KEY} --sender-nonce 0 --receiver-amount 1000 --zkapp-account-key ${ZKAPP_ACCOUNT_KEY} --fee 5 | sed 1,7d)
+send_zkapp_txn "${ZKAPP_TXN_QUERY}"
+
 # Unlock Genesis Accounts
 echo "==================== UNLOCKING GENESIS ACCOUNTS ======================"
 mina accounts unlock --public-key $BLOCK_PRODUCER_PK
 mina accounts unlock --public-key $SNARK_PRODUCER_PK
 
-# Start sending payments
-send_payments() {
+
+# Start sending value transfer transactions
+send_value_transfer_txns() {
   mina client send-payment -rest-server http://127.0.0.1:${MINA_GRAPHQL_PORT}/graphql -amount 1 -nonce 0 -receiver $BLOCK_PRODUCER_PK -sender $BLOCK_PRODUCER_PK
   while true; do
     sleep $TRANSACTION_FREQUENCY
     mina client send-payment -rest-server http://127.0.0.1:${MINA_GRAPHQL_PORT}/graphql -amount 1 -receiver $BLOCK_PRODUCER_PK -sender $BLOCK_PRODUCER_PK
   done
 }
-send_payments &
-
-# Fee payer cache creation
-echo "==================== PREPARE FEE PAYER CACHE ======================"
-zkapp_fee_payer_pk=$(cat ${MINA_KEYS_PATH}/zkapp-${zkapp}-fee-payer.key.pub)
-zkapp_fee_payer_privkey=$(mina advanced dump-keypair --privkey-path "${MINA_KEYS_PATH}/zkapp-${zkapp}-fee-payer.key" | sed -ne "s/Private key: //p")
-
-mkdir -p /root/.cache/zkapp-cli/keys
-echo -e "{\n    \"privateKey\": \"${zkapp_fee_payer_privkey}\",\n    \"publicKey\": \"${zkapp_fee_payer_pk}\"\n}" >/root/.cache/zkapp-cli/keys/sandbox.json
-
-# Deploy zkApps
-echo "==================== DEPLOYING ZKAPPS ======================"
-echo "If this fails, it's likely due to incompatibility between the
-o1js and zkapp-cli versions in use."
-echo "NOTE: At the moment the daemon still has an old version of
-      snarkyjs pinned to it, so we cannot use the latest version of
-      zkapp-cli, which requires o1js. Once the pinned snarkyjs version
-      gets updated, this build will most likely fail and we will then
-      need to update the zkapp-cli version used here. This is
-      unfortunate, but necessary. Please delete this warning once it's
-      done." > /dev/stderr
-
-deploy_txs=()
-for zkapp_path in ${ZKAPP_PATH}/*/; do
-  zkapp_path=${zkapp_path%/}
-  zkapp=$(basename $zkapp_path)
-  echo "Deploying ${zkapp}..."
-
-  zkapp_account_pk=$(cat ${MINA_KEYS_PATH}/zkapp-${zkapp}-account.key.pub)
-  zkapp_account_privkey=$(mina advanced dump-keypair --privkey-path "${MINA_KEYS_PATH}/zkapp-${zkapp}-account.key" | sed -ne "s/Private key: //p")
-
-  mkdir -p ${zkapp_path}/keys
-  echo -e "{\n    \"privateKey\": \"${zkapp_account_privkey}\",\n    \"publicKey\": \"${zkapp_account_pk}\"\n}" >"${zkapp_path}/keys/sandbox.json"
-
-  cat <<EOF >"${zkapp_path}/config.json"
-{
-  "version": 1,
-  "networks": {
-    "sandbox": {
-      "url": "http://127.0.0.1:${MINA_GRAPHQL_PORT}/graphql",
-      "keyPath": "keys/sandbox.json",
-      "feepayerKeyPath": "/root/.cache/zkapp-cli/keys/sandbox.json",
-      "feepayerAlias": "sandbox",
-      "fee": "1"
-    }
-  }
-}
-EOF
-  cd "$zkapp_path"
-  npm ci
-  npm run build
-  txn=$(zk deploy sandbox -y | sed -ne "s/https:\/\/berkeley.minaexplorer.com\/transaction\///p")
-  deploy_txs+=txn
-  cd -
-  echo "Done."
-done
+send_value_transfer_txns &
 
-# TODO: wait until all zkApps deploy txns are included in a block
-
-next_block_time=$(mina client status --json | jq '.next_block_production.timing[1].time' | tr -d '"')
-curr_time=$(date +%s%N | cut -b1-13)
-sleep_time=$((($next_block_time - $curr_time) / 1000))
-echo "Sleeping for ${sleep_time}s until next block is created..."
-sleep ${sleep_time}
+# Start sending zkapp transactions
+ZKAPP_FEE_PAYER_NONCE=1
+ZKAPP_SENDER_NONCE=1
+ZKAPP_STATE=0
+send_zkapp_transactions() {
+  while true; do
+    ZKAPP_TXN_QUERY=$(mina-zkapp-test-transaction transfer-funds-one-receiver --fee-payer-key ${ZKAPP_FEE_PAYER_KEY} --nonce ${ZKAPP_FEE_PAYER_NONCE} --sender-key ${ZKAPP_SENDER_KEY} --sender-nonce ${ZKAPP_SENDER_NONCE} --receiver-amount 1 --fee 5 --receiver ${ZKAPP_ACCOUNT_PUB_KEY} | sed 1,5d)
+    send_zkapp_txn "${ZKAPP_TXN_QUERY}"
+    let ZKAPP_FEE_PAYER_NONCE++
+    let ZKAPP_SENDER_NONCE++
+
+    ZKAPP_TXN_QUERY=$(mina-zkapp-test-transaction update-state --fee-payer-key ${ZKAPP_FEE_PAYER_KEY} --nonce ${ZKAPP_FEE_PAYER_NONCE} --zkapp-account-key ${ZKAPP_SENDER_KEY} --zkapp-state ${ZKAPP_STATE} --fee 5 | sed 1,5d)
+    send_zkapp_txn "${ZKAPP_TXN_QUERY}"
+    let ZKAPP_FEE_PAYER_NONCE++
+    let ZKAPP_STATE++
+  done
+}
 
-# Start calling zkApp methods
-echo "==================== INTERACTING WITH ZKAPPS ======================"
-RECEIVER_PK=$BLOCK_PRODUCER_PK
-for zkapp_path in ${ZKAPP_PATH}/*/; do
-  zkapp_path=${zkapp_path%/}
-  zkapp=$(basename $zkapp_path)
-  echo "Interacting with ${zkapp}..."
-
-  cd "$zkapp_path"
-  ./interact.sh sandbox
-  cd -
-  echo "Done."
-done
+# There is no point to generate transaction just for minimal mode
+if [[ "$MODE" == "full" ]]; then
+  send_zkapp_transactions &
+fi
 
-next_block_time=$(mina client status --json | jq '.next_block_production.timing[1].time' | tr -d '"')
-curr_time=$(date +%s%N | cut -b1-13)
+next_block_time=$(mina client status --json | jq '.next_block_production.timing[1].time' | tr -d '"') curr_time=$(date +%s%N | cut -b1-13)
 sleep_time=$((($next_block_time - $curr_time) / 1000))
 echo "Sleeping for ${sleep_time}s until next block is created..."
 sleep ${sleep_time}
@@ -327,11 +242,15 @@ rosetta-cli configuration:validate ${ROSETTA_CONFIGURATION_FILE}
 echo "========================= ROSETTA CLI: CHECK:SPEC ==========================="
 rosetta-cli check:spec --all --configuration-file ${ROSETTA_CONFIGURATION_FILE}
 
-echo "========================= ROSETTA CLI: CHECK:CONSTRUCTION ==========================="
-rosetta-cli check:construction --configuration-file ${ROSETTA_CONFIGURATION_FILE}
+if [[ "$MODE" == "full" ]]; then
+
+  echo "========================= ROSETTA CLI: CHECK:CONSTRUCTION ==========================="
+  rosetta-cli check:construction --configuration-file ${ROSETTA_CONFIGURATION_FILE}
+
+  echo "========================= ROSETTA CLI: CHECK:DATA ==========================="
+  rosetta-cli check:data --configuration-file ${ROSETTA_CONFIGURATION_FILE}
 
-echo "========================= ROSETTA CLI: CHECK:DATA ==========================="
-rosetta-cli check:data --configuration-file ${ROSETTA_CONFIGURATION_FILE}
+  echo "========================= ROSETTA CLI: CHECK:PERF ==========================="
+  echo "rosetta-cli check:perf" # Will run this command when tests are fully implemented
 
-echo "========================= ROSETTA CLI: CHECK:PERF ==========================="
-echo "rosetta-cli check:perf" # Will run this command when tests are fully implemented
+fi

buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall

@@ -20,7 +20,8 @@ let dirtyWhen = [
   S.strictlyStart (S.contains "src/lib"),
   S.strictlyStart (S.contains "src/app/archive"),
   S.exactly "buildkite/src/Jobs/Test/RosettaIntegrationTests" "dhall",
-  S.exactly "buildkite/scripts/rosetta-integration-tests" "sh"
+  S.exactly "buildkite/scripts/rosetta-integration-tests" "sh",
+  S.exactly "buildkite/scripts/rosetta-integration-tests-fast" "sh"
 ]
 
 let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type
@@ -41,7 +42,7 @@ Pipeline.build
         Command.Config::{
           commands = [
             Cmd.run ("export MINA_DEB_CODENAME=bullseye && source ./buildkite/scripts/export-git-env-vars.sh && echo \\\${MINA_DOCKER_TAG}"),
-            Cmd.runInDocker Cmd.Docker::{image="gcr.io/o1labs-192920/mina-rosetta:\\\${MINA_DOCKER_TAG}", entrypoint=" --entrypoint buildkite/scripts/rosetta-integration-tests.sh"} "bash"
+            Cmd.runInDocker Cmd.Docker::{image="gcr.io/o1labs-192920/mina-rosetta:\\\${MINA_DOCKER_TAG}", entrypoint=" --entrypoint buildkite/scripts/rosetta-integration-tests-fast.sh"} "bash"
           ],
           label = "Rosetta integration tests Bullseye"
           , key = "rosetta-integration-tests-bullseye"

buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall

@@ -0,0 +1,53 @@
+let Prelude = ../../External/Prelude.dhall
+let B = ../../External/Buildkite.dhall
+
+let Cmd = ../../Lib/Cmds.dhall
+let S = ../../Lib/SelectFiles.dhall
+
+let Pipeline = ../../Pipeline/Dsl.dhall
+let PipelineMode = ../../Pipeline/Mode.dhall
+let JobSpec = ../../Pipeline/JobSpec.dhall
+
+let Command = ../../Command/Base.dhall
+let RunInToolchain = ../../Command/RunInToolchain.dhall
+let Size = ../../Command/Size.dhall
+let Libp2p = ../../Command/Libp2pHelperBuild.dhall
+let DockerImage = ../../Command/DockerImage.dhall
+let DebianVersions = ../../Constants/DebianVersions.dhall
+
+let dirtyWhen = [ 
+  S.strictlyStart (S.contains "src/app/rosetta"),
+  S.strictlyStart (S.contains "src/lib"),
+  S.strictlyStart (S.contains "src/app/archive"),
+  S.exactly "buildkite/src/Jobs/Test/RosettaIntegrationTests" "dhall",
+  S.exactly "buildkite/scripts/rosetta-integration-tests" "sh",
+  S.exactly "buildkite/scripts/rosetta-integration-tests-full" "sh"
+]
+
+let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type
+
+in
+
+Pipeline.build
+  Pipeline.Config::
+    { spec =
+      JobSpec::{
+        dirtyWhen = dirtyWhen,
+        path = "Test",
+        name = "RosettaIntegrationTestsLong",
+        mode = PipelineMode.Type.Stable
+      }
+    , steps = [
+      Command.build
+        Command.Config::{
+          commands = [
+            Cmd.run ("export MINA_DEB_CODENAME=bullseye && source ./buildkite/scripts/export-git-env-vars.sh && echo \\\${MINA_DOCKER_TAG}"),
+            Cmd.runInDocker Cmd.Docker::{image="gcr.io/o1labs-192920/mina-rosetta:\\\${MINA_DOCKER_TAG}", entrypoint=" --entrypoint buildkite/scripts/rosetta-integration-tests-full.sh"} "bash"
+          ],
+          label = "Rosetta integration tests Bullseye Long"
+          , key = "rosetta-integration-tests-bullseye-long"
+          , target = Size.Small
+          , depends_on = [ { name = "MinaArtifactBullseye", key = "rosetta-bullseye-docker-image" } ]
+        }
+    ]
+  }