Merge branch 'berkeley' into georgeee/orchestrator-on-aws

buildkite/scripts/build-artifact.sh

@@ -13,7 +13,7 @@ echo "--- Build libp2p_helper TODO: use the previously uploaded build artifact"
 make -C src/app/libp2p_helper
 
 MAINNET_TARGETS=""
-[[ ${MINA_BUILD_MAINNET} ]] && MAINNET_TARGETS="src/app/cli/src/mina_mainnet_signatures.exe src/app/rosetta/rosetta_mainnet_signatures.exe "
+[[ ${MINA_BUILD_MAINNET} ]] && MAINNET_TARGETS="src/app/cli/src/mina_mainnet_signatures.exe src/app/rosetta/rosetta_mainnet_signatures.exe src/app/rosetta/ocaml-signer/signer_mainnet_signatures.exe"
 
 echo "--- Build all major tagets required for packaging"
 echo "Building from Commit SHA: ${MINA_COMMIT_SHA1}"
@@ -33,6 +33,7 @@ dune build "--profile=${DUNE_PROFILE}" \
   src/app/swap_bad_balances/swap_bad_balances.exe \
   src/app/zkapp_test_transaction/zkapp_test_transaction.exe \
   src/app/rosetta/rosetta_testnet_signatures.exe \
+  src/app/rosetta/ocaml-signer/signer_testnet_signatures.exe \
   src/app/test_executive/test_executive.exe # 2>&1 | tee /tmp/buildocaml.log
 
 echo "--- Bundle all packages for Debian ${MINA_DEB_CODENAME}"

buildkite/scripts/rosetta-integration-tests.sh

@@ -12,7 +12,7 @@
 # that source. It is important to make sure that the zkapp-cli version installed
 # is compatible with o1js version used.
 
-# These tests use the mina-dev binary, as rosetta-cli assumes we use a testnet.
+# These tests use the mina binary, as rosetta-cli assumes we use a testnet.
 # See https://github.com/coinbase/rosetta-sdk-go/blob/master/keys/signer_pallas.go#L222
 
 set -eo pipefail
@@ -29,7 +29,7 @@ export PG_CONN=postgres://${POSTGRES_USERNAME}:${POSTGRES_USERNAME}@127.0.0.1:54
 
 # Mina Archive variables
 export MINA_ARCHIVE_PORT=${MINA_ARCHIVE_PORT:=3086}
-export MINA_ARCHIVE_SQL_SCHEMA_PATH=${MINA_ARCHIVE_SQL_SCHEMA_PATH:=/archive/create_schema.sql}
+export MINA_ARCHIVE_SQL_SCHEMA_PATH=${MINA_ARCHIVE_SQL_SCHEMA_PATH:=/etc/mina/rosetta/archive/create_schema.sql}
 
 # Mina Rosetta variables
 export MINA_ROSETTA_ONLINE_PORT=${MINA_ROSETTA_ONLINE_PORT:=3087}
@@ -88,7 +88,7 @@ popd
 # Files from ROSETTA_CLI_CONFIG_FILES will be read from
 # ROSETTA_CONFIGURATION_INPUT_DIR and some placeholders will be
 # substituted.
-ROSETTA_CONFIGURATION_INPUT_DIR=${ROSETTA_CONFIGURATION_INPUT_DIR:=/rosetta/rosetta-cli-config}
+ROSETTA_CONFIGURATION_INPUT_DIR=${ROSETTA_CONFIGURATION_INPUT_DIR:=/etc/mina/rosetta/rosetta-cli-config}
 ROSETTA_CLI_CONFIG_FILES=${ROSETTA_CLI_CONFIG_FILES:="config.json mina.ros"}
 ROSETTA_CLI_MAIN_CONFIG_FILE=${ROSETTA_CLI_MAIN_CONFIG_FILE:="config.json"}
 
@@ -101,13 +101,13 @@ mv /tmp/MinaProtocol-rosetta-integration-test-zkapps-* $ZKAPP_PATH
 
 # Libp2p Keypair
 echo "=========================== GENERATING KEYPAIR IN ${MINA_LIBP2P_KEYPAIR_PATH} ==========================="
-mina-dev libp2p generate-keypair -privkey-path $MINA_LIBP2P_KEYPAIR_PATH
+mina libp2p generate-keypair -privkey-path $MINA_LIBP2P_KEYPAIR_PATH
 
 # Configuration
 echo "=========================== GENERATING GENESIS LEDGER FOR ${MINA_NETWORK} ==========================="
 mkdir -p $MINA_KEYS_PATH
-mina-dev advanced generate-keypair --privkey-path $MINA_KEYS_PATH/block-producer.key
-mina-dev advanced generate-keypair --privkey-path $MINA_KEYS_PATH/snark-producer.key
+mina advanced generate-keypair --privkey-path $MINA_KEYS_PATH/block-producer.key
+mina advanced generate-keypair --privkey-path $MINA_KEYS_PATH/snark-producer.key
 chmod -R 0700 $MINA_KEYS_PATH
 BLOCK_PRODUCER_PK=$(cat $MINA_KEYS_PATH/block-producer.key.pub)
 SNARK_PRODUCER_PK=$(cat $MINA_KEYS_PATH/snark-producer.key.pub)
@@ -132,9 +132,9 @@ for zkapp_path in ${ZKAPP_PATH}/*/; do
   zkapp_path=${zkapp_path%/}
   zkapp=$(basename $zkapp_path)
   # Generate zkApp account keypair
-  mina-dev advanced generate-keypair --privkey-path ${MINA_KEYS_PATH}/zkapp-${zkapp}-account.key
+  mina advanced generate-keypair --privkey-path ${MINA_KEYS_PATH}/zkapp-${zkapp}-account.key
   # Generate zkApp fee payer keypair
-  mina-dev advanced generate-keypair --privkey-path ${MINA_KEYS_PATH}/zkapp-${zkapp}-fee-payer.key
+  mina advanced generate-keypair --privkey-path ${MINA_KEYS_PATH}/zkapp-${zkapp}-fee-payer.key
   zkapp_fee_payer_pk=$(cat $MINA_KEYS_PATH/zkapp-${zkapp}-fee-payer.key.pub)
   line="[{ \"pk\": \"${zkapp_fee_payer_pk}\", \"balance\": \"10000\", \"delegate\": null, \"sk\": null }]"
   jq ".ledger.accounts |= . + ${line}" $MINA_CONFIG_FILE >${MINA_CONFIG_FILE}.tmp
@@ -158,19 +158,19 @@ done
 
 # Import Genesis Accounts
 echo "==================== IMPORTING GENESIS ACCOUNTS ======================"
-mina-dev accounts import --privkey-path $MINA_KEYS_PATH/block-producer.key --config-directory $MINA_CONFIG_DIR
-mina-dev accounts import --privkey-path $MINA_KEYS_PATH/snark-producer.key --config-directory $MINA_CONFIG_DIR
+mina accounts import --privkey-path $MINA_KEYS_PATH/block-producer.key --config-directory $MINA_CONFIG_DIR
+mina accounts import --privkey-path $MINA_KEYS_PATH/snark-producer.key --config-directory $MINA_CONFIG_DIR
 for zkapp_path in ${ZKAPP_PATH}/*/; do
   zkapp_path=${zkapp_path%/}
   zkapp=$(basename $zkapp_path)
-  mina-dev accounts import --privkey-path $MINA_KEYS_PATH/zkapp-${zkapp}-fee-payer.key --config-directory $MINA_CONFIG_DIR
+  mina accounts import --privkey-path $MINA_KEYS_PATH/zkapp-${zkapp}-fee-payer.key --config-directory $MINA_CONFIG_DIR
 done
 
 # Postgres
 echo "========================= INITIALIZING POSTGRESQL ==========================="
 pg_ctlcluster ${POSTGRES_VERSION} main start
 pg_dropcluster --stop ${POSTGRES_VERSION} main
-pg_createcluster --start -d ${POSTGRES_DATA_DIR} --createclusterconf /rosetta/postgresql.conf ${POSTGRES_VERSION} main
+pg_createcluster --start -d ${POSTGRES_DATA_DIR} --createclusterconf /etc/mina/rosetta/postgresql.conf ${POSTGRES_VERSION} main
 sudo -u postgres psql --command "CREATE USER ${POSTGRES_USERNAME} WITH SUPERUSER PASSWORD '${POSTGRES_USERNAME}';"
 sudo -u postgres createdb -O ${POSTGRES_USERNAME} ${POSTGRES_DBNAME}
 psql -f "${MINA_ARCHIVE_SQL_SCHEMA_PATH}" "${PG_CONN}"
@@ -179,7 +179,7 @@ psql -f "${MINA_ARCHIVE_SQL_SCHEMA_PATH}" "${PG_CONN}"
 echo "=========================== STARTING ROSETTA API ONLINE AND OFFLINE INSTANCES ==========================="
 ports=($MINA_ROSETTA_ONLINE_PORT $MINA_ROSETTA_OFFLINE_PORT)
 for port in ${ports[*]}; do
-  mina-rosetta-dev \
+  mina-rosetta \
     --archive-uri "${PG_CONN}" \
     --graphql-uri http://127.0.0.1:${MINA_GRAPHQL_PORT}/graphql \
     --log-level ${LOG_LEVEL} \
@@ -198,7 +198,7 @@ sleep 5
 
 # Daemon
 echo "========================= STARTING DAEMON connected to ${MINA_NETWORK} ==========================="
-mina-dev daemon \
+mina daemon \
   --archive-address 127.0.0.1:${MINA_ARCHIVE_PORT} \
   --background \
   --block-producer-pubkey "$BLOCK_PRODUCER_PK" \
@@ -218,29 +218,29 @@ retries_left=20
 until [ $daemon_status == "Synced" ]; do
   [[ $retries_left -eq 0 ]] && echo "Unable to Sync the Daemon" && exit 1 || ((retries_left--))
   sleep 15
-  daemon_status=$(mina-dev client status --json | jq -r .sync_status 2>/dev/null || echo "Pending")
+  daemon_status=$(mina client status --json | jq -r .sync_status 2>/dev/null || echo "Pending")
   echo "Daemon Status: ${daemon_status}"
 done
 
 # Unlock Genesis Accounts
 echo "==================== UNLOCKING GENESIS ACCOUNTS ======================"
-mina-dev accounts unlock --public-key $BLOCK_PRODUCER_PK
-mina-dev accounts unlock --public-key $SNARK_PRODUCER_PK
+mina accounts unlock --public-key $BLOCK_PRODUCER_PK
+mina accounts unlock --public-key $SNARK_PRODUCER_PK
 
 # Start sending payments
 send_payments() {
-  mina-dev client send-payment -rest-server http://127.0.0.1:${MINA_GRAPHQL_PORT}/graphql -amount 1 -nonce 0 -receiver $BLOCK_PRODUCER_PK -sender $BLOCK_PRODUCER_PK
+  mina client send-payment -rest-server http://127.0.0.1:${MINA_GRAPHQL_PORT}/graphql -amount 1 -nonce 0 -receiver $BLOCK_PRODUCER_PK -sender $BLOCK_PRODUCER_PK
   while true; do
     sleep $TRANSACTION_FREQUENCY
-    mina-dev client send-payment -rest-server http://127.0.0.1:${MINA_GRAPHQL_PORT}/graphql -amount 1 -receiver $BLOCK_PRODUCER_PK -sender $BLOCK_PRODUCER_PK
+    mina client send-payment -rest-server http://127.0.0.1:${MINA_GRAPHQL_PORT}/graphql -amount 1 -receiver $BLOCK_PRODUCER_PK -sender $BLOCK_PRODUCER_PK
   done
 }
 send_payments &
 
 # Fee payer cache creation
 echo "==================== PREPARE FEE PAYER CACHE ======================"
 zkapp_fee_payer_pk=$(cat ${MINA_KEYS_PATH}/zkapp-${zkapp}-fee-payer.key.pub)
-zkapp_fee_payer_privkey=$(mina-dev advanced dump-keypair --privkey-path "${MINA_KEYS_PATH}/zkapp-${zkapp}-fee-payer.key" | sed -ne "s/Private key: //p")
+zkapp_fee_payer_privkey=$(mina advanced dump-keypair --privkey-path "${MINA_KEYS_PATH}/zkapp-${zkapp}-fee-payer.key" | sed -ne "s/Private key: //p")
 
 mkdir -p /root/.cache/zkapp-cli/keys
 echo -e "{\n    \"privateKey\": \"${zkapp_fee_payer_privkey}\",\n    \"publicKey\": \"${zkapp_fee_payer_pk}\"\n}" >/root/.cache/zkapp-cli/keys/sandbox.json
@@ -264,7 +264,7 @@ for zkapp_path in ${ZKAPP_PATH}/*/; do
   echo "Deploying ${zkapp}..."
 
   zkapp_account_pk=$(cat ${MINA_KEYS_PATH}/zkapp-${zkapp}-account.key.pub)
-  zkapp_account_privkey=$(mina-dev advanced dump-keypair --privkey-path "${MINA_KEYS_PATH}/zkapp-${zkapp}-account.key" | sed -ne "s/Private key: //p")
+  zkapp_account_privkey=$(mina advanced dump-keypair --privkey-path "${MINA_KEYS_PATH}/zkapp-${zkapp}-account.key" | sed -ne "s/Private key: //p")
 
   mkdir -p ${zkapp_path}/keys
   echo -e "{\n    \"privateKey\": \"${zkapp_account_privkey}\",\n    \"publicKey\": \"${zkapp_account_pk}\"\n}" >"${zkapp_path}/keys/sandbox.json"
@@ -294,7 +294,7 @@ done
 
 # TODO: wait until all zkApps deploy txns are included in a block
 
-next_block_time=$(mina-dev client status --json | jq '.next_block_production.timing[1].time' | tr -d '"')
+next_block_time=$(mina client status --json | jq '.next_block_production.timing[1].time' | tr -d '"')
 curr_time=$(date +%s%N | cut -b1-13)
 sleep_time=$((($next_block_time - $curr_time) / 1000))
 echo "Sleeping for ${sleep_time}s until next block is created..."
@@ -314,7 +314,7 @@ for zkapp_path in ${ZKAPP_PATH}/*/; do
   echo "Done."
 done
 
-next_block_time=$(mina-dev client status --json | jq '.next_block_production.timing[1].time' | tr -d '"')
+next_block_time=$(mina client status --json | jq '.next_block_production.timing[1].time' | tr -d '"')
 curr_time=$(date +%s%N | cut -b1-13)
 sleep_time=$((($next_block_time - $curr_time) / 1000))
 echo "Sleeping for ${sleep_time}s until next block is created..."

buildkite/src/Command/MinaArtifact.dhall

@@ -99,8 +99,9 @@ let pipeline : DebianVersions.DebVersion -> PipelineMode.Type -> Pipeline.Config
 
         -- rosetta image
         let rosettaSpec = DockerImage.ReleaseSpec::{
+          deps=DebianVersions.dependsOn debVersion,
           service="mina-rosetta",
-          extra_args="--build-arg MINA_BRANCH=\\\${BUILDKITE_BRANCH} --cache-from ${DebianVersions.toolchainImage debVersion}",
+          network="berkeley",
           deb_codename="${DebianVersions.lowerName debVersion}",
           step_key="rosetta-${DebianVersions.lowerName debVersion}-docker-image"
         }

dockerfiles/stages/4-production → dockerfiles/Dockerfile-mina-rosetta

@@ -1,13 +1,13 @@
-#################################################################################################
-# The "production" Stage
-# - sets up the final container with built binaries and a running postgresql archive node setup
-#################################################################################################
-# Supports debian:buster-slim, debian:bullseye-slim, and ubuntu:focal
 ARG image=ubuntu:focal
 FROM ${image} AS production
 ARG deb_codename=focal
+ARG deb_version
+ARG deb_release=unstable
+ARG network=mainnet
 
 ARG psql_version=13
+# Golang version number used to detemine tarball name
+ARG GO_VERSION=1.19.11
 
 ARG MINA_DAEMON_PORT=10101
 ARG MINA_CONFIG_DIR=/data/.mina-config
@@ -18,6 +18,23 @@ ARG ROSETTA_CLI_VERSION=v0.10.1
 
 ENV DEBIAN_FRONTEND noninteractive
 
+# We do not install the below platform-specific dependencies,
+# and instead assume that apt will install the proper deps based on the package definition.
+# The packages are noted here just for clarity/documentation.
+# Stretch-only Deps:
+#    libffi6
+#    libprocps6
+#    libjemalloc1
+# Buster-only Deps:
+#    libffi7
+#    libprocps7
+#    libjemalloc2
+# Bullseye/Focal-only Deps:
+#    libffi7
+#    libprocps8
+#    libjemalloc2
+
+
 # --- Dependencies across many platforms
 RUN apt-get update --quiet --yes \
   && apt-get upgrade --quiet --yes \
@@ -26,6 +43,7 @@ RUN apt-get update --quiet --yes \
     apt-transport-https \
     curl \
     ca-certificates \
+    build-essential \
     dnsutils \
     dumb-init \
     gettext \
@@ -46,41 +64,14 @@ RUN curl -s https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \
        postgresql-"$psql_version" \
        postgresql-client-"$psql_version"
 
-# --- Debian 11 "Bullseye" Dependencies
-RUN test "$deb_codename" = "bullseye" \
-     && apt-get update -y \
-     && apt-get install -y \
-         libjemalloc2 \
-         libprocps8 \
-         libffi7 \
-     || exit 0
-
-# --- Debian 10 "Buster" Dependencies
-RUN test "$deb_codename" = "buster" \
-     && apt-get update --quiet --yes \
-     && apt-get install --quiet --yes --no-install-recommends \
-         libjemalloc2 \
-         libprocps7 \
-         libffi6 \
-     || exit 0
-
-# --- Ubuntu 20.04 "Focal Fossa" Dependencies
-RUN test "$deb_codename" = "focal" \
-     && apt-get update --quiet --yes \
-     && apt-get install --quiet --yes --no-install-recommends \
-         libjemalloc2 \
-         libprocps8 \
-         libffi7 \
-     || exit 0
+# --- Golang install of a given GO_VERSION (add -v for spam output of each file from the go dist)
+RUN curl -s "https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz" | tar -xz -C /usr/lib/
 
 # --- Install rosetta-cli
-
 # The following commands install rosetta-cli with an updated rosetta-sdk-go dependency, which supports delegation transactions.
 # Upstream PR to rosetta-sdk-go: https://github.com/coinbase/rosetta-sdk-go/pull/457
 # They can be replaced once there is a new release of rosetta-cli containing the above change by:
-# RUN curl -sSfL https://raw.githubusercontent.com/coinbase/rosetta-cli/${ROSETTA_CLI_VERSION}/scripts/install.sh | sh -s
-COPY --from=builder /usr/lib/go /usr/lib/go
-RUN apt-get install -y gcc
+#RUN curl -sSfL https://raw.githubusercontent.com/coinbase/rosetta-cli/${ROSETTA_CLI_VERSION}/scripts/install.sh | sh -s
 RUN export GOBIN="$(pwd)/bin" \
   && curl -L "https://github.com/coinbase/rosetta-cli/archive/refs/tags/v0.10.1.tar.gz" -o "/tmp/v0.10.1.tar.gz" \
   && tar xzf "/tmp/v0.10.1.tar.gz" -C "/tmp" \
@@ -99,15 +90,14 @@ RUN mkdir -p --mode=700 ${MINA_CONFIG_DIR}/wallets/store/ \
   > ${MINA_CONFIG_DIR}/wallets/store/${PK} \
   && chmod go-rwx ${MINA_CONFIG_DIR}/wallets/store/${PK}
 
-# --- Copy artifacts from the 3-builder stage
-COPY --from=builder /home/opam/app/* /usr/local/bin/
-COPY --from=builder /home/opam/mina/src/app/rosetta/*.sh /rosetta/
-COPY --from=builder /home/opam/mina/src/app/rosetta/*.conf /rosetta/
-COPY --from=builder /home/opam/mina/src/app/rosetta/README.md /rosetta/
-COPY --from=builder /home/opam/mina/src/app/rosetta/rosetta-cli-config/*.json /rosetta/rosetta-cli-config/
-COPY --from=builder /home/opam/mina/src/app/rosetta/rosetta-cli-config/*.ros /rosetta/rosetta-cli-config/
-COPY --from=builder /home/opam/mina/src/app/archive/*.sql /archive/
-COPY --from=builder /home/opam/mina/genesis_ledgers /genesis_ledgers/
+
+# Mina daemon package
+# jemalloc is also installed automatically here to match the package dependencies for this $deb_codename
+RUN echo "Building image with version $deb_version from repo $deb_release $deb_codename for network $network" \
+  && echo "deb [trusted=yes] http://packages.o1test.net $deb_codename $deb_release" > /etc/apt/sources.list.d/o1.list \
+  && apt-get update --quiet --yes \
+  && apt-get install --quiet --yes --allow-downgrades "mina-$network=$deb_version" "mina-archive=$deb_version"  \
+  && rm -rf /var/lib/apt/lists/*
 
 # --- Set up postgres
 USER postgres
@@ -118,10 +108,10 @@ RUN POSTGRES_VERSION=$(psql -V | cut -d " " -f 3 | sed 's/.[[:digit:]]*$//g') \
 # Run as root so it can create /data/postgresql
 USER root
 RUN POSTGRES_VERSION=$(psql -V | cut -d " " -f 3 | sed 's/.[[:digit:]]*$//g') \
-  && pg_createcluster --start -d /data/postgresql --createclusterconf /rosetta/postgresql.conf ${POSTGRES_VERSION} main
+  && pg_createcluster --start -d /data/postgresql --createclusterconf /etc/mina/rosetta/postgresql.conf ${POSTGRES_VERSION} main
 
 # --- Container workdir, ports, entrypoint, etc.
-WORKDIR /rosetta
+WORKDIR /etc/mina/rosetta
 
 EXPOSE 3087
 EXPOSE $MINA_DAEMON_PORT