From eab593394aea14c7eee4732c2325dc4fd36b9078 Mon Sep 17 00:00:00 2001 From: dkijania Date: Fri, 6 Oct 2023 20:52:18 +0200 Subject: [PATCH 01/22] New rosetta docker --- buildkite/scripts/build-artifact.sh | 3 +- .../4-production => Dockerfile-mina-rosetta} | 85 +++++++++++-------- dockerfiles/stages/3-builder | 72 ---------------- dockerfiles/stages/4-deb-builder | 58 ------------- scripts/rebuild-deb.sh | 3 +- scripts/release-docker.sh | 2 +- src/app/rosetta/rosetta_testnet_signatures.ml | 2 +- 7 files changed, 55 insertions(+), 170 deletions(-) rename dockerfiles/{stages/4-production => Dockerfile-mina-rosetta} (68%) mode change 100644 => 100755 delete mode 100644 dockerfiles/stages/3-builder delete mode 100644 dockerfiles/stages/4-deb-builder diff --git a/buildkite/scripts/build-artifact.sh b/buildkite/scripts/build-artifact.sh index adb98a79ff8..3af63a0e1dc 100755 --- a/buildkite/scripts/build-artifact.sh +++ b/buildkite/scripts/build-artifact.sh @@ -13,7 +13,7 @@ echo "--- Build libp2p_helper TODO: use the previously uploaded build artifact" make -C src/app/libp2p_helper MAINNET_TARGETS="" -[[ ${MINA_BUILD_MAINNET} ]] && MAINNET_TARGETS="src/app/cli/src/mina_mainnet_signatures.exe src/app/rosetta/rosetta_mainnet_signatures.exe " +[[ ${MINA_BUILD_MAINNET} ]] && MAINNET_TARGETS="src/app/cli/src/mina_mainnet_signatures.exe src/app/rosetta/rosetta_mainnet_signatures.exe src/app/rosetta/ocaml-signer/signer_mainnet_signatures.exe" echo "--- Build all major tagets required for packaging" echo "Building from Commit SHA: ${MINA_COMMIT_SHA1}" @@ -33,6 +33,7 @@ dune build "--profile=${DUNE_PROFILE}" \ src/app/swap_bad_balances/swap_bad_balances.exe \ src/app/zkapp_test_transaction/zkapp_test_transaction.exe \ src/app/rosetta/rosetta_testnet_signatures.exe \ + src/app/rosetta/ocaml-signer/signer_testnet_signatures.exe \ src/app/test_executive/test_executive.exe # 2>&1 | tee /tmp/buildocaml.log echo "--- Bundle all packages for Debian ${MINA_DEB_CODENAME}" diff --git a/dockerfiles/stages/4-production b/dockerfiles/Dockerfile-mina-rosetta old mode 100644 new mode 100755 similarity index 68% rename from dockerfiles/stages/4-production rename to dockerfiles/Dockerfile-mina-rosetta index 1e45c124f74..b4eaecbca06 --- a/dockerfiles/stages/4-production +++ b/dockerfiles/Dockerfile-mina-rosetta @@ -6,6 +6,10 @@ ARG image=ubuntu:focal FROM ${image} AS production ARG deb_codename=focal +ARG deb_version +ARG deb_release=unstable +ARG network=mainnet + ARG psql_version=13 @@ -18,6 +22,23 @@ ARG ROSETTA_CLI_VERSION=v0.10.1 ENV DEBIAN_FRONTEND noninteractive +# We do not install the below platform-specific dependencies, +# and instead assume that apt will install the proper deps based on the package definition. +# The packages are noted here just for clarity/documentation. +# Stretch-only Deps: +# libffi6 +# libprocps6 +# libjemalloc1 +# Buster-only Deps: +# libffi7 +# libprocps7 +# libjemalloc2 +# Bullseye/Focal-only Deps: +# libffi7 +# libprocps8 +# libjemalloc2 + + # --- Dependencies across many platforms RUN apt-get update --quiet --yes \ && apt-get upgrade --quiet --yes \ @@ -46,33 +67,6 @@ RUN curl -s https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \ postgresql-"$psql_version" \ postgresql-client-"$psql_version" -# --- Debian 11 "Bullseye" Dependencies -RUN test "$deb_codename" = "bullseye" \ - && apt-get update -y \ - && apt-get install -y \ - libjemalloc2 \ - libprocps8 \ - libffi7 \ - || exit 0 - -# --- Debian 10 "Buster" Dependencies -RUN test "$deb_codename" = "buster" \ - && apt-get update --quiet --yes \ - && apt-get install --quiet --yes --no-install-recommends \ - libjemalloc2 \ - libprocps7 \ - libffi6 \ - || exit 0 - -# --- Ubuntu 20.04 "Focal Fossa" Dependencies -RUN test "$deb_codename" = "focal" \ - && apt-get update --quiet --yes \ - && apt-get install --quiet --yes --no-install-recommends \ - libjemalloc2 \ - libprocps8 \ - libffi7 \ - || exit 0 - # --- Install rosetta-cli # The following commands install rosetta-cli with an updated rosetta-sdk-go dependency, which supports delegation transactions. @@ -89,6 +83,16 @@ RUN export GOBIN="$(pwd)/bin" \ && /usr/lib/go/bin/go mod tidy \ && /usr/lib/go/bin/go install + +# Rosetta-demo artifacts / build dir removal: +# Make rosetta-crucial components and the generate_keypair tool +# && dune build --profile=${DUNE_PROFILE} \ +# && _build/default/src/app/runtime_genesis_ledger/runtime_genesis_ledger.exe \ +# --config-file src/app/rosetta/demo-config.json \ +# --genesis-dir ${HOME}/demo-genesis \ +# && mv _build/default/src/app $HOME/app \ +# && rm -rf _build + # --- Generate en_US.UTF-8 locale to allow use of O(1) Labs DB dumps RUN locale-gen en_US.UTF-8 @@ -99,15 +103,24 @@ RUN mkdir -p --mode=700 ${MINA_CONFIG_DIR}/wallets/store/ \ > ${MINA_CONFIG_DIR}/wallets/store/${PK} \ && chmod go-rwx ${MINA_CONFIG_DIR}/wallets/store/${PK} -# --- Copy artifacts from the 3-builder stage -COPY --from=builder /home/opam/app/* /usr/local/bin/ -COPY --from=builder /home/opam/mina/src/app/rosetta/*.sh /rosetta/ -COPY --from=builder /home/opam/mina/src/app/rosetta/*.conf /rosetta/ -COPY --from=builder /home/opam/mina/src/app/rosetta/README.md /rosetta/ -COPY --from=builder /home/opam/mina/src/app/rosetta/rosetta-cli-config/*.json /rosetta/rosetta-cli-config/ -COPY --from=builder /home/opam/mina/src/app/rosetta/rosetta-cli-config/*.ros /rosetta/rosetta-cli-config/ -COPY --from=builder /home/opam/mina/src/app/archive/*.sql /archive/ -COPY --from=builder /home/opam/mina/genesis_ledgers /genesis_ledgers/ + +# Mina daemon package +# jemalloc is also installed automatically here to match the package dependencies for this $deb_codename +RUN echo "Building image with version $deb_version from repo $deb_release $deb_codename for network $network" \ + && echo "deb [trusted=yes] http://packages.o1test.net $deb_codename $deb_release" > /etc/apt/sources.list.d/o1.list \ + && apt-get update --quiet --yes \ + && apt-get install --quiet --yes --allow-downgrades "mina-$network=$deb_version" \ + && rm -rf /var/lib/apt/lists/* + + +# --- Copy artifacts +COPY src/app/rosetta/*.sh /rosetta/ +COPY src/app/rosetta/*.conf /rosetta/ +COPY src/app/rosetta/README.md /rosetta/ +COPY src/app/rosetta/rosetta-cli-config/*.json /rosetta/rosetta-cli-config/ +COPY src/app/rosetta/rosetta-cli-config/*.ros /rosetta/rosetta-cli-config/ +COPY src/app/archive/*.sql /archive/ +COPY genesis_ledgers /genesis_ledgers/ # --- Set up postgres USER postgres diff --git a/dockerfiles/stages/3-builder b/dockerfiles/stages/3-builder deleted file mode 100644 index 38b36f5d066..00000000000 --- a/dockerfiles/stages/3-builder +++ /dev/null @@ -1,72 +0,0 @@ -################################################################################################# -# The "builder" Stage -# - builds mina and any other binaries required to run rosetta -# - should not include any data related to joining a specific network, only the node software itself -################################################################################################# -FROM opam-deps AS builder - -# Use --build-arg "DUNE_PROFILE=dev" to build a dev image or for CI -ARG DUNE_PROFILE=devnet - -# branch to checkout on first clone (this will be the only available branch in the container) -# can also be a tagged release -ARG MINA_BRANCH=compatible - -# repo to checkout the branch from -ARG MINA_REPO=https://github.com/MinaProtocol/mina - -# location of repo used for pins and external package commits -ARG MINA_DIR=mina - -ENV PATH "$PATH:/usr/lib/go/bin:$HOME/.cargo/bin" - -# --- Shallow Clone the Mina repo -# Delete any previous clones of the mina repo and then clone into an empty directory -RUN cd $HOME && rm -rf $HOME/${MINA_DIR} \ - && git clone \ - -b "${MINA_BRANCH}" \ - --depth 1 \ - --shallow-submodules \ - --recurse-submodules \ - ${MINA_REPO} ${HOME}/${MINA_DIR} -# Set the working directory to the cloned mina repo -WORKDIR $HOME/${MINA_DIR} -# Create a directory for compiled outputs -RUN mkdir ${HOME}/app - -# --- Build libp2p_helper -RUN make libp2p_helper \ - && mv src/app/libp2p_helper/result/bin/libp2p_helper ${HOME}/app/libp2p_helper - -# --- Make rosetta-crucial components and the generate_keypair tool -RUN eval $(opam config env) \ - && dune build --profile=${DUNE_PROFILE} \ - src/app/cli/src/mina_testnet_signatures.exe \ - src/app/cli/src/mina_mainnet_signatures.exe \ - src/app/archive/archive.exe \ - src/app/archive_blocks/archive_blocks.exe \ - src/app/extract_blocks/extract_blocks.exe \ - src/app/missing_blocks_auditor/missing_blocks_auditor.exe \ - src/app/replayer/replayer.exe \ - src/app/rosetta/rosetta_testnet_signatures.exe \ - src/app/rosetta/rosetta_mainnet_signatures.exe \ - src/app/generate_keypair/generate_keypair.exe \ - src/app/validate_keypair/validate_keypair.exe \ - src/app/rosetta/ocaml-signer/signer.exe \ - && cp _build/default/src/app/archive_blocks/archive_blocks.exe $HOME/app/mina-archive-blocks \ - && cp _build/default/src/app/extract_blocks/extract_blocks.exe $HOME/app/mina-extract-blocks \ - && cp _build/default/src/app/missing_blocks_auditor/missing_blocks_auditor.exe $HOME/app/mina-missing-blocks-auditor \ - && cp _build/default/src/app/replayer/replayer.exe $HOME/app/mina-replayer \ - && mv _build/default/src/app/cli/src/mina_mainnet_signatures.exe $HOME/app/mina \ - && mv _build/default/src/app/cli/src/mina_testnet_signatures.exe $HOME/app/mina-dev \ - && mv _build/default/src/app/archive/archive.exe $HOME/app/mina-archive \ - && mv _build/default/src/app/rosetta/rosetta_mainnet_signatures.exe $HOME/app/mina-rosetta \ - && mv _build/default/src/app/rosetta/rosetta_testnet_signatures.exe $HOME/app/mina-rosetta-dev \ - && mv _build/default/src/app/generate_keypair/generate_keypair.exe $HOME/app/mina-generate-keypair \ - && mv _build/default/src/app/validate_keypair/validate_keypair.exe $HOME/app/mina-validate-keypair \ - && mv _build/default/src/app/rosetta/ocaml-signer/signer.exe $HOME/app/mina-ocaml-signer \ - && rm -rf _build - -# --- Clear go module caches to make the container smaller -RUN cd src/app/libp2p_helper/src/libp2p_helper \ - && go clean --cache --modcache --testcache -r diff --git a/dockerfiles/stages/4-deb-builder b/dockerfiles/stages/4-deb-builder deleted file mode 100644 index 38dbf10a2cf..00000000000 --- a/dockerfiles/stages/4-deb-builder +++ /dev/null @@ -1,58 +0,0 @@ -################################################################################################# -# The "deb-builder" Stage -# - builds mina and any other binaries required to run `make deb` -################################################################################################# -FROM toolchain AS deb-builder - -# Use --build-arg "DUNE_PROFILE=dev" to build a dev image or for CI -ARG DUNE_PROFILE=devnet - -# branch to checkout on first clone (this will be the only available branch in the container) -# can also be a tagged release -ARG MINA_BRANCH=compatible - -ARG MINA_COMMIT - -# repo to checkout the branch from -ARG MINA_REPO=https://github.com/MinaProtocol/mina - -# location of repo used for pins and external package commits -ARG MINA_DIR=mina - -ENV MINA_COMMIT_SHA1=${MINA_COMMIT} - -ENV PATH "$PATH:/usr/lib/go/bin:$HOME/.cargo/bin" - -# --- Build all major tagets required for packaging -RUN echo "--- Build all major tagets required for packaging" \ - && echo "Building from Commit SHA: ${MINA_COMMIT_SHA1}" \ - && eval $(opam config env) \ - && dune build --profile=${DUNE_PROFILE} \ - src/app/logproc/logproc.exe \ - src/app/runtime_genesis_ledger/runtime_genesis_ledger.exe \ - src/app/generate_keypair/generate_keypair.exe \ - src/app/validate_keypair/validate_keypair.exe \ - src/app/cli/src/mina.exe \ - src/app/cli/src/mina_testnet_signatures.exe \ - src/app/cli/src/mina_mainnet_signatures.exe \ - src/app/archive/archive.exe \ - src/app/archive/archive_testnet_signatures.exe \ - src/app/archive/archive_mainnet_signatures.exe \ - src/app/rosetta/rosetta.exe \ - src/app/rosetta/rosetta_mainnet_signatures.exe \ - src/app/rosetta/rosetta_testnet_signatures.exe \ - src/app/rosetta/ocaml-signer/signer.exe - -# --- MAKE DEB -RUN eval $(opam config env) \ - && echo "Building from Commit SHA: ${MINA_COMMIT_SHA1}" \ - && ./scripts/rebuild-deb.sh - -# Rosetta-demo artifacts / build dir removal: -# Make rosetta-crucial components and the generate_keypair tool -# && dune build --profile=${DUNE_PROFILE} \ -# && _build/default/src/app/runtime_genesis_ledger/runtime_genesis_ledger.exe \ -# --config-file src/app/rosetta/demo-config.json \ -# --genesis-dir ${HOME}/demo-genesis \ -# && mv _build/default/src/app $HOME/app \ -# && rm -rf _build diff --git a/scripts/rebuild-deb.sh b/scripts/rebuild-deb.sh index 770e1dcbf60..ea1e48894af 100755 --- a/scripts/rebuild-deb.sh +++ b/scripts/rebuild-deb.sh @@ -131,7 +131,8 @@ copy_common_daemon_configs() { # Copy signature-based Binaries (based on signature type $2 passed into the function) cp ./default/src/app/cli/src/mina_${2}_signatures.exe "${BUILDDIR}/usr/local/bin/mina" cp ./default/src/app/rosetta/rosetta_${2}_signatures.exe "${BUILDDIR}/usr/local/bin/mina-rosetta" - + cp ./default/src/app/rosetta/ocaml-signer/signer_${2}_signatures.exe "${BUILDDIR}/usr/local/bin/mina-rosetta-ocaml-signer" + # Copy over Build Configs (based on $2) mkdir -p "${BUILDDIR}/etc/coda/build_config" # Use parameter expansion to either return "mainnet.mlh" or "devnet.mlh" diff --git a/scripts/release-docker.sh b/scripts/release-docker.sh index 4407af6656f..cbf5b26289c 100755 --- a/scripts/release-docker.sh +++ b/scripts/release-docker.sh @@ -95,7 +95,7 @@ mina-batch-txn) VERSION="${VERSION}-${NETWORK##*=}" ;; mina-rosetta) - DOCKERFILE_PATH="dockerfiles/stages/1-build-deps dockerfiles/stages/2-opam-deps dockerfiles/stages/3-builder dockerfiles/stages/4-production" + DOCKERFILE_PATH="dockerfiles/Dockerfile-mina-rosetta" ;; mina-zkapp-test-transaction) DOCKERFILE_PATH="dockerfiles/Dockerfile-zkapp-test-transaction" diff --git a/src/app/rosetta/rosetta_testnet_signatures.ml b/src/app/rosetta/rosetta_testnet_signatures.ml index ca32b93398a..b7a558d7145 100644 --- a/src/app/rosetta/rosetta_testnet_signatures.ml +++ b/src/app/rosetta/rosetta_testnet_signatures.ml @@ -3,4 +3,4 @@ open Async let () = Command.run - (Command.async ~summary:"Run Rosetta process on top of Coda" command) + (Command.async ~summary:"Run Rosetta process on top of Mina" command) From 498533f0589cd3688a3227ab0e262da513799f79 Mon Sep 17 00:00:00 2001 From: dkijania Date: Fri, 6 Oct 2023 21:03:14 +0200 Subject: [PATCH 02/22] comp fix --- buildkite/scripts/build-artifact.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/scripts/build-artifact.sh b/buildkite/scripts/build-artifact.sh index 3af63a0e1dc..bdcc48d4c1e 100755 --- a/buildkite/scripts/build-artifact.sh +++ b/buildkite/scripts/build-artifact.sh @@ -33,7 +33,7 @@ dune build "--profile=${DUNE_PROFILE}" \ src/app/swap_bad_balances/swap_bad_balances.exe \ src/app/zkapp_test_transaction/zkapp_test_transaction.exe \ src/app/rosetta/rosetta_testnet_signatures.exe \ - src/app/rosetta/ocaml-signer/signer_testnet_signatures.exe \ + src/app/rosetta/ocaml-signer/signer_testnet_signatures.exe \ src/app/test_executive/test_executive.exe # 2>&1 | tee /tmp/buildocaml.log echo "--- Bundle all packages for Debian ${MINA_DEB_CODENAME}" From 9459d39cc31a08ede354965fe70e94a86e63f6fa Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 9 Oct 2023 11:22:17 +0200 Subject: [PATCH 03/22] fix from --- dockerfiles/Dockerfile-mina-rosetta | 1 - 1 file changed, 1 deletion(-) diff --git a/dockerfiles/Dockerfile-mina-rosetta b/dockerfiles/Dockerfile-mina-rosetta index b4eaecbca06..47a79074d1a 100755 --- a/dockerfiles/Dockerfile-mina-rosetta +++ b/dockerfiles/Dockerfile-mina-rosetta @@ -73,7 +73,6 @@ RUN curl -s https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \ # Upstream PR to rosetta-sdk-go: https://github.com/coinbase/rosetta-sdk-go/pull/457 # They can be replaced once there is a new release of rosetta-cli containing the above change by: # RUN curl -sSfL https://raw.githubusercontent.com/coinbase/rosetta-cli/${ROSETTA_CLI_VERSION}/scripts/install.sh | sh -s -COPY --from=builder /usr/lib/go /usr/lib/go RUN apt-get install -y gcc RUN export GOBIN="$(pwd)/bin" \ && curl -L "https://github.com/coinbase/rosetta-cli/archive/refs/tags/v0.10.1.tar.gz" -o "/tmp/v0.10.1.tar.gz" \ From 435d430ce6fa0ceb49510dbe93e65b7f54790cb1 Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 9 Oct 2023 22:38:33 +0200 Subject: [PATCH 04/22] fix rosetta docker --- dockerfiles/Dockerfile-mina-rosetta | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/dockerfiles/Dockerfile-mina-rosetta b/dockerfiles/Dockerfile-mina-rosetta index 47a79074d1a..b74ae5819eb 100755 --- a/dockerfiles/Dockerfile-mina-rosetta +++ b/dockerfiles/Dockerfile-mina-rosetta @@ -72,15 +72,15 @@ RUN curl -s https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \ # The following commands install rosetta-cli with an updated rosetta-sdk-go dependency, which supports delegation transactions. # Upstream PR to rosetta-sdk-go: https://github.com/coinbase/rosetta-sdk-go/pull/457 # They can be replaced once there is a new release of rosetta-cli containing the above change by: -# RUN curl -sSfL https://raw.githubusercontent.com/coinbase/rosetta-cli/${ROSETTA_CLI_VERSION}/scripts/install.sh | sh -s -RUN apt-get install -y gcc -RUN export GOBIN="$(pwd)/bin" \ - && curl -L "https://github.com/coinbase/rosetta-cli/archive/refs/tags/v0.10.1.tar.gz" -o "/tmp/v0.10.1.tar.gz" \ - && tar xzf "/tmp/v0.10.1.tar.gz" -C "/tmp" \ - && cd "/tmp/rosetta-cli-0.10.1" \ - && /usr/lib/go/bin/go mod edit -replace github.com/coinbase/rosetta-sdk-go@v0.8.1=github.com/MinaProtocol/rosetta-sdk-go@stake-delegation-v1 \ - && /usr/lib/go/bin/go mod tidy \ - && /usr/lib/go/bin/go install +RUN curl -sSfL https://raw.githubusercontent.com/coinbase/rosetta-cli/${ROSETTA_CLI_VERSION}/scripts/install.sh | sh -s +#RUN apt-get install -y golang-go +#RUN export GOBIN="$(pwd)/bin" \ +## && curl -L "https://github.com/coinbase/rosetta-cli/archive/refs/tags/v0.10.1.tar.gz" -o "/tmp/v0.10.1.tar.gz" \ + # && tar xzf "/tmp/v0.10.1.tar.gz" -C "/tmp" \ +# && cd "/tmp/rosetta-cli-0.10.1" \ +# && /usr/lib/go/bin/go mod edit -replace github.com/coinbase/rosetta-sdk-go@v0.8.1=github.com/MinaProtocol/rosetta-sdk-go@stake-delegation-v1 \ +# && /usr/lib/go/bin/go mod tidy \ +# && /usr/lib/go/bin/go install # Rosetta-demo artifacts / build dir removal: @@ -115,11 +115,11 @@ RUN echo "Building image with version $deb_version from repo $deb_release $deb_c # --- Copy artifacts COPY src/app/rosetta/*.sh /rosetta/ COPY src/app/rosetta/*.conf /rosetta/ -COPY src/app/rosetta/README.md /rosetta/ +COPY src/app/rosetta/*.md /rosetta/ COPY src/app/rosetta/rosetta-cli-config/*.json /rosetta/rosetta-cli-config/ COPY src/app/rosetta/rosetta-cli-config/*.ros /rosetta/rosetta-cli-config/ COPY src/app/archive/*.sql /archive/ -COPY genesis_ledgers /genesis_ledgers/ +COPY genesis_ledgers/* /genesis_ledgers/ # --- Set up postgres USER postgres From 71cf6958ab315c5b435124a716ca76b3a17f248b Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 10 Oct 2023 00:27:33 +0200 Subject: [PATCH 05/22] better deb --- dockerfiles/Dockerfile-mina-rosetta | 13 +++---------- scripts/rebuild-deb.sh | 26 +++++++++++++++++++++++++- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/dockerfiles/Dockerfile-mina-rosetta b/dockerfiles/Dockerfile-mina-rosetta index b74ae5819eb..4b65b3c30d6 100755 --- a/dockerfiles/Dockerfile-mina-rosetta +++ b/dockerfiles/Dockerfile-mina-rosetta @@ -108,18 +108,11 @@ RUN mkdir -p --mode=700 ${MINA_CONFIG_DIR}/wallets/store/ \ RUN echo "Building image with version $deb_version from repo $deb_release $deb_codename for network $network" \ && echo "deb [trusted=yes] http://packages.o1test.net $deb_codename $deb_release" > /etc/apt/sources.list.d/o1.list \ && apt-get update --quiet --yes \ - && apt-get install --quiet --yes --allow-downgrades "mina-$network=$deb_version" \ + && apt-get install --quiet --yes --allow-downgrades "mina-$network=$deb_version" "mina-rosetta=$deb_version" \ && rm -rf /var/lib/apt/lists/* - -# --- Copy artifacts -COPY src/app/rosetta/*.sh /rosetta/ -COPY src/app/rosetta/*.conf /rosetta/ -COPY src/app/rosetta/*.md /rosetta/ -COPY src/app/rosetta/rosetta-cli-config/*.json /rosetta/rosetta-cli-config/ -COPY src/app/rosetta/rosetta-cli-config/*.ros /rosetta/rosetta-cli-config/ -COPY src/app/archive/*.sql /archive/ -COPY genesis_ledgers/* /genesis_ledgers/ +COPY /etc/mina/rosetta /rosetta +COPY /etc/mina/rosetta/genesis_ledgers /genesis_ledgers/ # --- Set up postgres USER postgres diff --git a/scripts/rebuild-deb.sh b/scripts/rebuild-deb.sh index ea1e48894af..baa1a1f2d51 100755 --- a/scripts/rebuild-deb.sh +++ b/scripts/rebuild-deb.sh @@ -260,7 +260,31 @@ cp ./default/src/app/zkapp_test_transaction/zkapp_test_transaction.exe "${BUILDD build_deb mina-zkapp-test-transaction -##################################### END SNAPP TEST TXN PACKAGE ####################################### +##################################### END ZKAPP TEST TXN PACKAGE ####################################### + +##################################### ROSETTA ####################################### +echo "------------------------------------------------------------" +echo "--- Building Mina Rosetta :" + +create_control_file mina-rosetta "${SHARED_DEPS}${DAEMON_DEPS}" 'Rosetta implementation for Mina' + +# --- Copy artifacts +cp "../src/app/rosetta/*.conf" "${BUILDDIR}/etc/mina/rosetta" +cp "../src/app/rosetta/*.sh" "${BUILDDIR}/etc/mina/rosetta" +cp "../src/app/rosetta/*.sh" "${BUILDDIR}/etc/mina/rosetta" + +cp ../src/app/rosetta/rosetta-cli-config/*.json "${BUILDDIR}/etc/mina/rosetta/rosetta-cli-config" +cp ../src/app/rosetta/rosetta-cli-config/*.ros "${BUILDDIR}/etc/mina/rosetta/rosetta-cli-config" +cp ../src/app/archive/*.sql "${BUILDDIR}/etc/mina/rosetta/archive" +cp ../genesis_ledgers/* ${BUILDDIR}/etc/mina/rosetta/genesis_ledgers/ + +cp ./default/src/app/rosetta/rosetta_${2}_signatures.exe "${BUILDDIR}/usr/local/bin/mina-rosetta" +cp ./default/src/app/rosetta/ocaml-signer/signer_${2}_signatures.exe "${BUILDDIR}/usr/local/bin/mina-rosetta-ocaml-signer" + +build_deb mina-rosetta + +#################################### END ROSETTA PACKAGE ####################################### + ##################################### BERKELEY PACKAGE ####################################### echo "------------------------------------------------------------" From 20cd9f7d7f1efef2797cac055e7cb27dbbe37f48 Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 10 Oct 2023 09:26:00 +0200 Subject: [PATCH 06/22] clean deb job and dependency --- buildkite/src/Command/MinaArtifact.dhall | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/src/Command/MinaArtifact.dhall b/buildkite/src/Command/MinaArtifact.dhall index 89ed25e662b..4b9b2a592b6 100644 --- a/buildkite/src/Command/MinaArtifact.dhall +++ b/buildkite/src/Command/MinaArtifact.dhall @@ -93,8 +93,8 @@ let pipeline : DebianVersions.DebVersion -> Pipeline.Config.Type = \(debVersion -- rosetta image let rosettaSpec = DockerImage.ReleaseSpec::{ + deps=DebianVersions.dependsOn debVersion, service="mina-rosetta", - extra_args="--build-arg MINA_BRANCH=\\\${BUILDKITE_BRANCH} --cache-from ${DebianVersions.toolchainImage debVersion}", deb_codename="${DebianVersions.lowerName debVersion}", step_key="rosetta-${DebianVersions.lowerName debVersion}-docker-image" } From 1ba559b64852b243b13d10c0ca587e9b43556847 Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 10 Oct 2023 10:25:10 +0200 Subject: [PATCH 07/22] fix cp --- scripts/rebuild-deb.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scripts/rebuild-deb.sh b/scripts/rebuild-deb.sh index baa1a1f2d51..9dbe2a5b4a0 100755 --- a/scripts/rebuild-deb.sh +++ b/scripts/rebuild-deb.sh @@ -269,14 +269,13 @@ echo "--- Building Mina Rosetta :" create_control_file mina-rosetta "${SHARED_DEPS}${DAEMON_DEPS}" 'Rosetta implementation for Mina' # --- Copy artifacts -cp "../src/app/rosetta/*.conf" "${BUILDDIR}/etc/mina/rosetta" -cp "../src/app/rosetta/*.sh" "${BUILDDIR}/etc/mina/rosetta" -cp "../src/app/rosetta/*.sh" "${BUILDDIR}/etc/mina/rosetta" +cp ../src/app/rosetta/*.conf "${BUILDDIR}/etc/mina/rosetta" +cp ../src/app/rosetta/*.sh" "${BUILDDIR}/etc/mina/rosetta cp ../src/app/rosetta/rosetta-cli-config/*.json "${BUILDDIR}/etc/mina/rosetta/rosetta-cli-config" cp ../src/app/rosetta/rosetta-cli-config/*.ros "${BUILDDIR}/etc/mina/rosetta/rosetta-cli-config" cp ../src/app/archive/*.sql "${BUILDDIR}/etc/mina/rosetta/archive" -cp ../genesis_ledgers/* ${BUILDDIR}/etc/mina/rosetta/genesis_ledgers/ +cp -r ../genesis_ledgers/* ${BUILDDIR}/etc/mina/rosetta/genesis_ledgers/ cp ./default/src/app/rosetta/rosetta_${2}_signatures.exe "${BUILDDIR}/usr/local/bin/mina-rosetta" cp ./default/src/app/rosetta/ocaml-signer/signer_${2}_signatures.exe "${BUILDDIR}/usr/local/bin/mina-rosetta-ocaml-signer" From ecf81213f244bfca1efcde1520e53dcc70752321 Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 10 Oct 2023 10:50:10 +0200 Subject: [PATCH 08/22] create dirs --- scripts/rebuild-deb.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/rebuild-deb.sh b/scripts/rebuild-deb.sh index 9dbe2a5b4a0..eeb0b98872a 100755 --- a/scripts/rebuild-deb.sh +++ b/scripts/rebuild-deb.sh @@ -268,9 +268,15 @@ echo "--- Building Mina Rosetta :" create_control_file mina-rosetta "${SHARED_DEPS}${DAEMON_DEPS}" 'Rosetta implementation for Mina' +mkdir -p "${BUILDDIR}/etc/mina/rosetta" +mkdir -p "${BUILDDIR}/etc/mina/rosetta/rosetta-cli-config" +mkdir -p "${BUILDDIR}/usr/local/bin/mina-rosetta" +mkdir -p "${BUILDDIR}/etc/mina/rosetta/archive" +mkdir -p "${BUILDDIR}/etc/mina/rosetta/genesis_ledgers" + # --- Copy artifacts cp ../src/app/rosetta/*.conf "${BUILDDIR}/etc/mina/rosetta" -cp ../src/app/rosetta/*.sh" "${BUILDDIR}/etc/mina/rosetta +cp ../src/app/rosetta/*.sh "${BUILDDIR}/etc/mina/rosetta" cp ../src/app/rosetta/rosetta-cli-config/*.json "${BUILDDIR}/etc/mina/rosetta/rosetta-cli-config" cp ../src/app/rosetta/rosetta-cli-config/*.ros "${BUILDDIR}/etc/mina/rosetta/rosetta-cli-config" From d9ec4f0daac31d87704c9d78441ea37a9920cb96 Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 10 Oct 2023 18:07:28 +0200 Subject: [PATCH 09/22] set variable --- scripts/rebuild-deb.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/rebuild-deb.sh b/scripts/rebuild-deb.sh index eeb0b98872a..68b2bf8d014 100755 --- a/scripts/rebuild-deb.sh +++ b/scripts/rebuild-deb.sh @@ -283,8 +283,8 @@ cp ../src/app/rosetta/rosetta-cli-config/*.ros "${BUILDDIR}/etc/mina/rosetta/ros cp ../src/app/archive/*.sql "${BUILDDIR}/etc/mina/rosetta/archive" cp -r ../genesis_ledgers/* ${BUILDDIR}/etc/mina/rosetta/genesis_ledgers/ -cp ./default/src/app/rosetta/rosetta_${2}_signatures.exe "${BUILDDIR}/usr/local/bin/mina-rosetta" -cp ./default/src/app/rosetta/ocaml-signer/signer_${2}_signatures.exe "${BUILDDIR}/usr/local/bin/mina-rosetta-ocaml-signer" +cp ./default/src/app/rosetta/rosetta_testnet_signatures.exe "${BUILDDIR}/usr/local/bin/mina-rosetta" +cp ./default/src/app/rosetta/ocaml-signer/signer_testnet_signatures.exe "${BUILDDIR}/usr/local/bin/mina-rosetta-ocaml-signer" build_deb mina-rosetta From dc4223407e0913edd5405c8edff8e36f630e9229 Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 10 Oct 2023 18:40:27 +0200 Subject: [PATCH 10/22] fix network --- buildkite/src/Command/MinaArtifact.dhall | 1 + 1 file changed, 1 insertion(+) diff --git a/buildkite/src/Command/MinaArtifact.dhall b/buildkite/src/Command/MinaArtifact.dhall index 4b9b2a592b6..db16d9ba705 100644 --- a/buildkite/src/Command/MinaArtifact.dhall +++ b/buildkite/src/Command/MinaArtifact.dhall @@ -95,6 +95,7 @@ let pipeline : DebianVersions.DebVersion -> Pipeline.Config.Type = \(debVersion let rosettaSpec = DockerImage.ReleaseSpec::{ deps=DebianVersions.dependsOn debVersion, service="mina-rosetta", + network="berkeley", deb_codename="${DebianVersions.lowerName debVersion}", step_key="rosetta-${DebianVersions.lowerName debVersion}-docker-image" } From 35476478b5d8b13cf15ddc2e7ece6c8dcc094a52 Mon Sep 17 00:00:00 2001 From: dkijania Date: Wed, 11 Oct 2023 15:44:49 +0200 Subject: [PATCH 11/22] join rosetta with mina --- scripts/rebuild-deb.sh | 45 +++++++++++++++--------------------------- 1 file changed, 16 insertions(+), 29 deletions(-) diff --git a/scripts/rebuild-deb.sh b/scripts/rebuild-deb.sh index 68b2bf8d014..d5a7923108d 100755 --- a/scripts/rebuild-deb.sh +++ b/scripts/rebuild-deb.sh @@ -130,9 +130,25 @@ copy_common_daemon_configs() { # Copy signature-based Binaries (based on signature type $2 passed into the function) cp ./default/src/app/cli/src/mina_${2}_signatures.exe "${BUILDDIR}/usr/local/bin/mina" + + # Copy rosetta-based Binaries cp ./default/src/app/rosetta/rosetta_${2}_signatures.exe "${BUILDDIR}/usr/local/bin/mina-rosetta" cp ./default/src/app/rosetta/ocaml-signer/signer_${2}_signatures.exe "${BUILDDIR}/usr/local/bin/mina-rosetta-ocaml-signer" + mkdir -p "${BUILDDIR}/etc/mina/rosetta" + mkdir -p "${BUILDDIR}/etc/mina/rosetta/rosetta-cli-config" + mkdir -p "${BUILDDIR}/etc/mina/rosetta/archive" + mkdir -p "${BUILDDIR}/etc/mina/rosetta/genesis_ledgers" + + # --- Copy artifacts + cp ../src/app/rosetta/*.conf "${BUILDDIR}/etc/mina/rosetta" + cp ../src/app/rosetta/*.sh "${BUILDDIR}/etc/mina/rosetta" + + cp ../src/app/rosetta/rosetta-cli-config/*.json "${BUILDDIR}/etc/mina/rosetta/rosetta-cli-config" + cp ../src/app/rosetta/rosetta-cli-config/*.ros "${BUILDDIR}/etc/mina/rosetta/rosetta-cli-config" + cp ../src/app/archive/*.sql "${BUILDDIR}/etc/mina/rosetta/archive" + cp -r ../genesis_ledgers/* ${BUILDDIR}/etc/mina/rosetta/genesis_ledgers/ + # Copy over Build Configs (based on $2) mkdir -p "${BUILDDIR}/etc/coda/build_config" # Use parameter expansion to either return "mainnet.mlh" or "devnet.mlh" @@ -262,35 +278,6 @@ build_deb mina-zkapp-test-transaction ##################################### END ZKAPP TEST TXN PACKAGE ####################################### -##################################### ROSETTA ####################################### -echo "------------------------------------------------------------" -echo "--- Building Mina Rosetta :" - -create_control_file mina-rosetta "${SHARED_DEPS}${DAEMON_DEPS}" 'Rosetta implementation for Mina' - -mkdir -p "${BUILDDIR}/etc/mina/rosetta" -mkdir -p "${BUILDDIR}/etc/mina/rosetta/rosetta-cli-config" -mkdir -p "${BUILDDIR}/usr/local/bin/mina-rosetta" -mkdir -p "${BUILDDIR}/etc/mina/rosetta/archive" -mkdir -p "${BUILDDIR}/etc/mina/rosetta/genesis_ledgers" - -# --- Copy artifacts -cp ../src/app/rosetta/*.conf "${BUILDDIR}/etc/mina/rosetta" -cp ../src/app/rosetta/*.sh "${BUILDDIR}/etc/mina/rosetta" - -cp ../src/app/rosetta/rosetta-cli-config/*.json "${BUILDDIR}/etc/mina/rosetta/rosetta-cli-config" -cp ../src/app/rosetta/rosetta-cli-config/*.ros "${BUILDDIR}/etc/mina/rosetta/rosetta-cli-config" -cp ../src/app/archive/*.sql "${BUILDDIR}/etc/mina/rosetta/archive" -cp -r ../genesis_ledgers/* ${BUILDDIR}/etc/mina/rosetta/genesis_ledgers/ - -cp ./default/src/app/rosetta/rosetta_testnet_signatures.exe "${BUILDDIR}/usr/local/bin/mina-rosetta" -cp ./default/src/app/rosetta/ocaml-signer/signer_testnet_signatures.exe "${BUILDDIR}/usr/local/bin/mina-rosetta-ocaml-signer" - -build_deb mina-rosetta - -#################################### END ROSETTA PACKAGE ####################################### - - ##################################### BERKELEY PACKAGE ####################################### echo "------------------------------------------------------------" echo "--- Building Mina Berkeley testnet signatures deb without keys:" From 47738a021966a2bea3a4ab2a6c659094a78de864 Mon Sep 17 00:00:00 2001 From: dkijania Date: Thu, 12 Oct 2023 13:30:31 +0200 Subject: [PATCH 12/22] do not download mina-rosetta --- dockerfiles/Dockerfile-mina-rosetta | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerfiles/Dockerfile-mina-rosetta b/dockerfiles/Dockerfile-mina-rosetta index 4b65b3c30d6..68ba76d81bc 100755 --- a/dockerfiles/Dockerfile-mina-rosetta +++ b/dockerfiles/Dockerfile-mina-rosetta @@ -108,7 +108,7 @@ RUN mkdir -p --mode=700 ${MINA_CONFIG_DIR}/wallets/store/ \ RUN echo "Building image with version $deb_version from repo $deb_release $deb_codename for network $network" \ && echo "deb [trusted=yes] http://packages.o1test.net $deb_codename $deb_release" > /etc/apt/sources.list.d/o1.list \ && apt-get update --quiet --yes \ - && apt-get install --quiet --yes --allow-downgrades "mina-$network=$deb_version" "mina-rosetta=$deb_version" \ + && apt-get install --quiet --yes --allow-downgrades "mina-$network=$deb_version" \ && rm -rf /var/lib/apt/lists/* COPY /etc/mina/rosetta /rosetta From 1907c7e67442d5275af4729b44ccdd4e39728437 Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 16 Oct 2023 19:57:23 +0200 Subject: [PATCH 13/22] remove copy --- dockerfiles/Dockerfile-mina-rosetta | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/dockerfiles/Dockerfile-mina-rosetta b/dockerfiles/Dockerfile-mina-rosetta index 68ba76d81bc..98db7fceb5d 100755 --- a/dockerfiles/Dockerfile-mina-rosetta +++ b/dockerfiles/Dockerfile-mina-rosetta @@ -111,9 +111,6 @@ RUN echo "Building image with version $deb_version from repo $deb_release $deb_c && apt-get install --quiet --yes --allow-downgrades "mina-$network=$deb_version" \ && rm -rf /var/lib/apt/lists/* -COPY /etc/mina/rosetta /rosetta -COPY /etc/mina/rosetta/genesis_ledgers /genesis_ledgers/ - # --- Set up postgres USER postgres RUN POSTGRES_VERSION=$(psql -V | cut -d " " -f 3 | sed 's/.[[:digit:]]*$//g') \ @@ -123,10 +120,10 @@ RUN POSTGRES_VERSION=$(psql -V | cut -d " " -f 3 | sed 's/.[[:digit:]]*$//g') \ # Run as root so it can create /data/postgresql USER root RUN POSTGRES_VERSION=$(psql -V | cut -d " " -f 3 | sed 's/.[[:digit:]]*$//g') \ - && pg_createcluster --start -d /data/postgresql --createclusterconf /rosetta/postgresql.conf ${POSTGRES_VERSION} main + && pg_createcluster --start -d /data/postgresql --createclusterconf /etc/mina/rosetta/postgresql.conf ${POSTGRES_VERSION} main # --- Container workdir, ports, entrypoint, etc. -WORKDIR /rosetta +WORKDIR /etc/mina/rosetta EXPOSE 3087 EXPOSE $MINA_DAEMON_PORT From ec49b33d4b31a3dacf9aaa2cf36e5e0505209b79 Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 16 Oct 2023 20:32:47 +0200 Subject: [PATCH 14/22] convert mina-dev mina --- .../scripts/rosetta-integration-tests.sh | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/buildkite/scripts/rosetta-integration-tests.sh b/buildkite/scripts/rosetta-integration-tests.sh index 1efca1562ef..75b59b3eb00 100755 --- a/buildkite/scripts/rosetta-integration-tests.sh +++ b/buildkite/scripts/rosetta-integration-tests.sh @@ -12,7 +12,7 @@ # that source. It is important to make sure that the zkapp-cli version installed # is compatible with o1js version used. -# These tests use the mina-dev binary, as rosetta-cli assumes we use a testnet. +# These tests use the mina binary, as rosetta-cli assumes we use a testnet. # See https://github.com/coinbase/rosetta-sdk-go/blob/master/keys/signer_pallas.go#L222 set -eo pipefail @@ -101,13 +101,13 @@ mv /tmp/MinaProtocol-rosetta-integration-test-zkapps-* $ZKAPP_PATH # Libp2p Keypair echo "=========================== GENERATING KEYPAIR IN ${MINA_LIBP2P_KEYPAIR_PATH} ===========================" -mina-dev libp2p generate-keypair -privkey-path $MINA_LIBP2P_KEYPAIR_PATH +mina libp2p generate-keypair -privkey-path $MINA_LIBP2P_KEYPAIR_PATH # Configuration echo "=========================== GENERATING GENESIS LEDGER FOR ${MINA_NETWORK} ===========================" mkdir -p $MINA_KEYS_PATH -mina-dev advanced generate-keypair --privkey-path $MINA_KEYS_PATH/block-producer.key -mina-dev advanced generate-keypair --privkey-path $MINA_KEYS_PATH/snark-producer.key +mina advanced generate-keypair --privkey-path $MINA_KEYS_PATH/block-producer.key +mina advanced generate-keypair --privkey-path $MINA_KEYS_PATH/snark-producer.key chmod -R 0700 $MINA_KEYS_PATH BLOCK_PRODUCER_PK=$(cat $MINA_KEYS_PATH/block-producer.key.pub) SNARK_PRODUCER_PK=$(cat $MINA_KEYS_PATH/snark-producer.key.pub) @@ -132,9 +132,9 @@ for zkapp_path in ${ZKAPP_PATH}/*/; do zkapp_path=${zkapp_path%/} zkapp=$(basename $zkapp_path) # Generate zkApp account keypair - mina-dev advanced generate-keypair --privkey-path ${MINA_KEYS_PATH}/zkapp-${zkapp}-account.key + mina advanced generate-keypair --privkey-path ${MINA_KEYS_PATH}/zkapp-${zkapp}-account.key # Generate zkApp fee payer keypair - mina-dev advanced generate-keypair --privkey-path ${MINA_KEYS_PATH}/zkapp-${zkapp}-fee-payer.key + mina advanced generate-keypair --privkey-path ${MINA_KEYS_PATH}/zkapp-${zkapp}-fee-payer.key zkapp_fee_payer_pk=$(cat $MINA_KEYS_PATH/zkapp-${zkapp}-fee-payer.key.pub) line="[{ \"pk\": \"${zkapp_fee_payer_pk}\", \"balance\": \"10000\", \"delegate\": null, \"sk\": null }]" jq ".ledger.accounts |= . + ${line}" $MINA_CONFIG_FILE >${MINA_CONFIG_FILE}.tmp @@ -158,12 +158,12 @@ done # Import Genesis Accounts echo "==================== IMPORTING GENESIS ACCOUNTS ======================" -mina-dev accounts import --privkey-path $MINA_KEYS_PATH/block-producer.key --config-directory $MINA_CONFIG_DIR -mina-dev accounts import --privkey-path $MINA_KEYS_PATH/snark-producer.key --config-directory $MINA_CONFIG_DIR +mina accounts import --privkey-path $MINA_KEYS_PATH/block-producer.key --config-directory $MINA_CONFIG_DIR +mina accounts import --privkey-path $MINA_KEYS_PATH/snark-producer.key --config-directory $MINA_CONFIG_DIR for zkapp_path in ${ZKAPP_PATH}/*/; do zkapp_path=${zkapp_path%/} zkapp=$(basename $zkapp_path) - mina-dev accounts import --privkey-path $MINA_KEYS_PATH/zkapp-${zkapp}-fee-payer.key --config-directory $MINA_CONFIG_DIR + mina accounts import --privkey-path $MINA_KEYS_PATH/zkapp-${zkapp}-fee-payer.key --config-directory $MINA_CONFIG_DIR done # Postgres @@ -198,7 +198,7 @@ sleep 5 # Daemon echo "========================= STARTING DAEMON connected to ${MINA_NETWORK} ===========================" -mina-dev daemon \ +mina daemon \ --archive-address 127.0.0.1:${MINA_ARCHIVE_PORT} \ --background \ --block-producer-pubkey "$BLOCK_PRODUCER_PK" \ @@ -218,21 +218,21 @@ retries_left=20 until [ $daemon_status == "Synced" ]; do [[ $retries_left -eq 0 ]] && echo "Unable to Sync the Daemon" && exit 1 || ((retries_left--)) sleep 15 - daemon_status=$(mina-dev client status --json | jq -r .sync_status 2>/dev/null || echo "Pending") + daemon_status=$(mina client status --json | jq -r .sync_status 2>/dev/null || echo "Pending") echo "Daemon Status: ${daemon_status}" done # Unlock Genesis Accounts echo "==================== UNLOCKING GENESIS ACCOUNTS ======================" -mina-dev accounts unlock --public-key $BLOCK_PRODUCER_PK -mina-dev accounts unlock --public-key $SNARK_PRODUCER_PK +mina accounts unlock --public-key $BLOCK_PRODUCER_PK +mina accounts unlock --public-key $SNARK_PRODUCER_PK # Start sending payments send_payments() { - mina-dev client send-payment -rest-server http://127.0.0.1:${MINA_GRAPHQL_PORT}/graphql -amount 1 -nonce 0 -receiver $BLOCK_PRODUCER_PK -sender $BLOCK_PRODUCER_PK + mina client send-payment -rest-server http://127.0.0.1:${MINA_GRAPHQL_PORT}/graphql -amount 1 -nonce 0 -receiver $BLOCK_PRODUCER_PK -sender $BLOCK_PRODUCER_PK while true; do sleep $TRANSACTION_FREQUENCY - mina-dev client send-payment -rest-server http://127.0.0.1:${MINA_GRAPHQL_PORT}/graphql -amount 1 -receiver $BLOCK_PRODUCER_PK -sender $BLOCK_PRODUCER_PK + mina client send-payment -rest-server http://127.0.0.1:${MINA_GRAPHQL_PORT}/graphql -amount 1 -receiver $BLOCK_PRODUCER_PK -sender $BLOCK_PRODUCER_PK done } send_payments & @@ -240,7 +240,7 @@ send_payments & # Fee payer cache creation echo "==================== PREPARE FEE PAYER CACHE ======================" zkapp_fee_payer_pk=$(cat ${MINA_KEYS_PATH}/zkapp-${zkapp}-fee-payer.key.pub) -zkapp_fee_payer_privkey=$(mina-dev advanced dump-keypair --privkey-path "${MINA_KEYS_PATH}/zkapp-${zkapp}-fee-payer.key" | sed -ne "s/Private key: //p") +zkapp_fee_payer_privkey=$(mina advanced dump-keypair --privkey-path "${MINA_KEYS_PATH}/zkapp-${zkapp}-fee-payer.key" | sed -ne "s/Private key: //p") mkdir -p /root/.cache/zkapp-cli/keys echo -e "{\n \"privateKey\": \"${zkapp_fee_payer_privkey}\",\n \"publicKey\": \"${zkapp_fee_payer_pk}\"\n}" >/root/.cache/zkapp-cli/keys/sandbox.json @@ -264,7 +264,7 @@ for zkapp_path in ${ZKAPP_PATH}/*/; do echo "Deploying ${zkapp}..." zkapp_account_pk=$(cat ${MINA_KEYS_PATH}/zkapp-${zkapp}-account.key.pub) - zkapp_account_privkey=$(mina-dev advanced dump-keypair --privkey-path "${MINA_KEYS_PATH}/zkapp-${zkapp}-account.key" | sed -ne "s/Private key: //p") + zkapp_account_privkey=$(mina advanced dump-keypair --privkey-path "${MINA_KEYS_PATH}/zkapp-${zkapp}-account.key" | sed -ne "s/Private key: //p") mkdir -p ${zkapp_path}/keys echo -e "{\n \"privateKey\": \"${zkapp_account_privkey}\",\n \"publicKey\": \"${zkapp_account_pk}\"\n}" >"${zkapp_path}/keys/sandbox.json" @@ -294,7 +294,7 @@ done # TODO: wait until all zkApps deploy txns are included in a block -next_block_time=$(mina-dev client status --json | jq '.next_block_production.timing[1].time' | tr -d '"') +next_block_time=$(mina client status --json | jq '.next_block_production.timing[1].time' | tr -d '"') curr_time=$(date +%s%N | cut -b1-13) sleep_time=$((($next_block_time - $curr_time) / 1000)) echo "Sleeping for ${sleep_time}s until next block is created..." @@ -314,7 +314,7 @@ for zkapp_path in ${ZKAPP_PATH}/*/; do echo "Done." done -next_block_time=$(mina-dev client status --json | jq '.next_block_production.timing[1].time' | tr -d '"') +next_block_time=$(mina client status --json | jq '.next_block_production.timing[1].time' | tr -d '"') curr_time=$(date +%s%N | cut -b1-13) sleep_time=$((($next_block_time - $curr_time) / 1000)) echo "Sleeping for ${sleep_time}s until next block is created..." From c7319893df122e640970142a103450b35db6e3f2 Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 16 Oct 2023 21:25:14 +0200 Subject: [PATCH 15/22] fix ocaml signer --- scripts/rebuild-deb.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/rebuild-deb.sh b/scripts/rebuild-deb.sh index d5a7923108d..8a5a9015cab 100755 --- a/scripts/rebuild-deb.sh +++ b/scripts/rebuild-deb.sh @@ -133,7 +133,7 @@ copy_common_daemon_configs() { # Copy rosetta-based Binaries cp ./default/src/app/rosetta/rosetta_${2}_signatures.exe "${BUILDDIR}/usr/local/bin/mina-rosetta" - cp ./default/src/app/rosetta/ocaml-signer/signer_${2}_signatures.exe "${BUILDDIR}/usr/local/bin/mina-rosetta-ocaml-signer" + cp ./default/src/app/rosetta/ocaml-signer/signer_${2}_signatures.exe "${BUILDDIR}/usr/local/bin/mina-ocaml-signer" mkdir -p "${BUILDDIR}/etc/mina/rosetta" mkdir -p "${BUILDDIR}/etc/mina/rosetta/rosetta-cli-config" From 95b5a7c8c7325a5db85b573e6776c994260e5424 Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 16 Oct 2023 22:49:49 +0200 Subject: [PATCH 16/22] fix path from /rosetta to /etc/mina/rosetta/ --- buildkite/scripts/rosetta-integration-tests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/buildkite/scripts/rosetta-integration-tests.sh b/buildkite/scripts/rosetta-integration-tests.sh index 75b59b3eb00..4075f28ab3e 100755 --- a/buildkite/scripts/rosetta-integration-tests.sh +++ b/buildkite/scripts/rosetta-integration-tests.sh @@ -88,7 +88,7 @@ popd # Files from ROSETTA_CLI_CONFIG_FILES will be read from # ROSETTA_CONFIGURATION_INPUT_DIR and some placeholders will be # substituted. -ROSETTA_CONFIGURATION_INPUT_DIR=${ROSETTA_CONFIGURATION_INPUT_DIR:=/rosetta/rosetta-cli-config} +ROSETTA_CONFIGURATION_INPUT_DIR=${ROSETTA_CONFIGURATION_INPUT_DIR:=/etc/mina/rosetta/rosetta-cli-config} ROSETTA_CLI_CONFIG_FILES=${ROSETTA_CLI_CONFIG_FILES:="config.json mina.ros"} ROSETTA_CLI_MAIN_CONFIG_FILE=${ROSETTA_CLI_MAIN_CONFIG_FILE:="config.json"} @@ -170,7 +170,7 @@ done echo "========================= INITIALIZING POSTGRESQL ===========================" pg_ctlcluster ${POSTGRES_VERSION} main start pg_dropcluster --stop ${POSTGRES_VERSION} main -pg_createcluster --start -d ${POSTGRES_DATA_DIR} --createclusterconf /rosetta/postgresql.conf ${POSTGRES_VERSION} main +pg_createcluster --start -d ${POSTGRES_DATA_DIR} --createclusterconf /etc/mina/rosetta/postgresql.conf ${POSTGRES_VERSION} main sudo -u postgres psql --command "CREATE USER ${POSTGRES_USERNAME} WITH SUPERUSER PASSWORD '${POSTGRES_USERNAME}';" sudo -u postgres createdb -O ${POSTGRES_USERNAME} ${POSTGRES_DBNAME} psql -f "${MINA_ARCHIVE_SQL_SCHEMA_PATH}" "${PG_CONN}" From 707a6c50ca769fe987a73d93092e7c97a6fb1eab Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 16 Oct 2023 23:39:53 +0200 Subject: [PATCH 17/22] fix archive path --- buildkite/scripts/rosetta-integration-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/scripts/rosetta-integration-tests.sh b/buildkite/scripts/rosetta-integration-tests.sh index 4075f28ab3e..6898dbc4186 100755 --- a/buildkite/scripts/rosetta-integration-tests.sh +++ b/buildkite/scripts/rosetta-integration-tests.sh @@ -29,7 +29,7 @@ export PG_CONN=postgres://${POSTGRES_USERNAME}:${POSTGRES_USERNAME}@127.0.0.1:54 # Mina Archive variables export MINA_ARCHIVE_PORT=${MINA_ARCHIVE_PORT:=3086} -export MINA_ARCHIVE_SQL_SCHEMA_PATH=${MINA_ARCHIVE_SQL_SCHEMA_PATH:=/archive/create_schema.sql} +export MINA_ARCHIVE_SQL_SCHEMA_PATH=${MINA_ARCHIVE_SQL_SCHEMA_PATH:=/etc/mina/rosetta/archive/create_schema.sql} # Mina Rosetta variables export MINA_ROSETTA_ONLINE_PORT=${MINA_ROSETTA_ONLINE_PORT:=3087} From c894d72555f5510f84790a4699a355382d8d1188 Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 17 Oct 2023 20:47:40 +0200 Subject: [PATCH 18/22] fix mina-rosetta name --- buildkite/scripts/rosetta-integration-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/scripts/rosetta-integration-tests.sh b/buildkite/scripts/rosetta-integration-tests.sh index 6898dbc4186..fb10d1e588a 100755 --- a/buildkite/scripts/rosetta-integration-tests.sh +++ b/buildkite/scripts/rosetta-integration-tests.sh @@ -179,7 +179,7 @@ psql -f "${MINA_ARCHIVE_SQL_SCHEMA_PATH}" "${PG_CONN}" echo "=========================== STARTING ROSETTA API ONLINE AND OFFLINE INSTANCES ===========================" ports=($MINA_ROSETTA_ONLINE_PORT $MINA_ROSETTA_OFFLINE_PORT) for port in ${ports[*]}; do - mina-rosetta-dev \ + mina-rosetta \ --archive-uri "${PG_CONN}" \ --graphql-uri http://127.0.0.1:${MINA_GRAPHQL_PORT}/graphql \ --log-level ${LOG_LEVEL} \ From ceabc2189226c9fe0841a5ae7740a8571fbd18c0 Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 17 Oct 2023 22:48:22 +0200 Subject: [PATCH 19/22] mina archive fix --- dockerfiles/Dockerfile-mina-rosetta | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerfiles/Dockerfile-mina-rosetta b/dockerfiles/Dockerfile-mina-rosetta index 98db7fceb5d..5b78a19ba98 100755 --- a/dockerfiles/Dockerfile-mina-rosetta +++ b/dockerfiles/Dockerfile-mina-rosetta @@ -108,7 +108,7 @@ RUN mkdir -p --mode=700 ${MINA_CONFIG_DIR}/wallets/store/ \ RUN echo "Building image with version $deb_version from repo $deb_release $deb_codename for network $network" \ && echo "deb [trusted=yes] http://packages.o1test.net $deb_codename $deb_release" > /etc/apt/sources.list.d/o1.list \ && apt-get update --quiet --yes \ - && apt-get install --quiet --yes --allow-downgrades "mina-$network=$deb_version" \ + && apt-get install --quiet --yes --allow-downgrades "mina-$network=$deb_version" "mina-archive=$deb_version" \ && rm -rf /var/lib/apt/lists/* # --- Set up postgres From a968932d3707fd9c3bc21e5081da3d841342938e Mon Sep 17 00:00:00 2001 From: dkijania Date: Wed, 18 Oct 2023 09:02:02 +0200 Subject: [PATCH 20/22] build rosetta instead of downloading --- dockerfiles/Dockerfile-mina-rosetta | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/dockerfiles/Dockerfile-mina-rosetta b/dockerfiles/Dockerfile-mina-rosetta index 5b78a19ba98..fc5b3b04e6b 100755 --- a/dockerfiles/Dockerfile-mina-rosetta +++ b/dockerfiles/Dockerfile-mina-rosetta @@ -72,15 +72,15 @@ RUN curl -s https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \ # The following commands install rosetta-cli with an updated rosetta-sdk-go dependency, which supports delegation transactions. # Upstream PR to rosetta-sdk-go: https://github.com/coinbase/rosetta-sdk-go/pull/457 # They can be replaced once there is a new release of rosetta-cli containing the above change by: -RUN curl -sSfL https://raw.githubusercontent.com/coinbase/rosetta-cli/${ROSETTA_CLI_VERSION}/scripts/install.sh | sh -s -#RUN apt-get install -y golang-go -#RUN export GOBIN="$(pwd)/bin" \ -## && curl -L "https://github.com/coinbase/rosetta-cli/archive/refs/tags/v0.10.1.tar.gz" -o "/tmp/v0.10.1.tar.gz" \ - # && tar xzf "/tmp/v0.10.1.tar.gz" -C "/tmp" \ -# && cd "/tmp/rosetta-cli-0.10.1" \ -# && /usr/lib/go/bin/go mod edit -replace github.com/coinbase/rosetta-sdk-go@v0.8.1=github.com/MinaProtocol/rosetta-sdk-go@stake-delegation-v1 \ -# && /usr/lib/go/bin/go mod tidy \ -# && /usr/lib/go/bin/go install +#RUN curl -sSfL https://raw.githubusercontent.com/coinbase/rosetta-cli/${ROSETTA_CLI_VERSION}/scripts/install.sh | sh -s +RUN apt-get install -y golang-go +RUN export GOBIN="$(pwd)/bin" \ + && curl -L "https://github.com/coinbase/rosetta-cli/archive/refs/tags/v0.10.1.tar.gz" -o "/tmp/v0.10.1.tar.gz" \ + && tar xzf "/tmp/v0.10.1.tar.gz" -C "/tmp" \ + && cd "/tmp/rosetta-cli-0.10.1" \ + && /usr/lib/go/bin/go mod edit -replace github.com/coinbase/rosetta-sdk-go@v0.8.1=github.com/MinaProtocol/rosetta-sdk-go@stake-delegation-v1 \ + && /usr/lib/go/bin/go mod tidy \ + && /usr/lib/go/bin/go install # Rosetta-demo artifacts / build dir removal: From d14db90d8ab07e1ff415eeb34dbd40273e7b92e1 Mon Sep 17 00:00:00 2001 From: dkijania Date: Wed, 18 Oct 2023 15:09:58 +0200 Subject: [PATCH 21/22] install gcc and go --- dockerfiles/Dockerfile-mina-rosetta | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/dockerfiles/Dockerfile-mina-rosetta b/dockerfiles/Dockerfile-mina-rosetta index fc5b3b04e6b..ac502bf46dd 100755 --- a/dockerfiles/Dockerfile-mina-rosetta +++ b/dockerfiles/Dockerfile-mina-rosetta @@ -12,6 +12,8 @@ ARG network=mainnet ARG psql_version=13 +# Golang version number used to detemine tarball name +ARG GO_VERSION=1.19.11 ARG MINA_DAEMON_PORT=10101 ARG MINA_CONFIG_DIR=/data/.mina-config @@ -47,6 +49,7 @@ RUN apt-get update --quiet --yes \ apt-transport-https \ curl \ ca-certificates \ + build-essential \ dnsutils \ dumb-init \ gettext \ @@ -69,11 +72,14 @@ RUN curl -s https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \ # --- Install rosetta-cli + +# --- Golang install of a given GO_VERSION (add -v for spam output of each file from the go dist) +RUN curl -s "https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz" | tar -xz -C /usr/lib/ + # The following commands install rosetta-cli with an updated rosetta-sdk-go dependency, which supports delegation transactions. # Upstream PR to rosetta-sdk-go: https://github.com/coinbase/rosetta-sdk-go/pull/457 # They can be replaced once there is a new release of rosetta-cli containing the above change by: #RUN curl -sSfL https://raw.githubusercontent.com/coinbase/rosetta-cli/${ROSETTA_CLI_VERSION}/scripts/install.sh | sh -s -RUN apt-get install -y golang-go RUN export GOBIN="$(pwd)/bin" \ && curl -L "https://github.com/coinbase/rosetta-cli/archive/refs/tags/v0.10.1.tar.gz" -o "/tmp/v0.10.1.tar.gz" \ && tar xzf "/tmp/v0.10.1.tar.gz" -C "/tmp" \ @@ -82,7 +88,6 @@ RUN export GOBIN="$(pwd)/bin" \ && /usr/lib/go/bin/go mod tidy \ && /usr/lib/go/bin/go install - # Rosetta-demo artifacts / build dir removal: # Make rosetta-crucial components and the generate_keypair tool # && dune build --profile=${DUNE_PROFILE} \ @@ -93,7 +98,7 @@ RUN export GOBIN="$(pwd)/bin" \ # && rm -rf _build # --- Generate en_US.UTF-8 locale to allow use of O(1) Labs DB dumps -RUN locale-gen en_US.UTF-8 +#RUN locale-gen en_US.UTF-8 # --- Set up sample/demo wallets and configuration RUN mkdir -p --mode=700 ${MINA_CONFIG_DIR}/wallets/store/ \ From 457ad030d4bdfbd78f337188d9045087d1265e20 Mon Sep 17 00:00:00 2001 From: dkijania Date: Wed, 18 Oct 2023 20:00:54 +0200 Subject: [PATCH 22/22] remove comments --- dockerfiles/Dockerfile-mina-rosetta | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/dockerfiles/Dockerfile-mina-rosetta b/dockerfiles/Dockerfile-mina-rosetta index ac502bf46dd..7b3e8cc78df 100755 --- a/dockerfiles/Dockerfile-mina-rosetta +++ b/dockerfiles/Dockerfile-mina-rosetta @@ -1,8 +1,3 @@ -################################################################################################# -# The "production" Stage -# - sets up the final container with built binaries and a running postgresql archive node setup -################################################################################################# -# Supports debian:buster-slim, debian:bullseye-slim, and ubuntu:focal ARG image=ubuntu:focal FROM ${image} AS production ARG deb_codename=focal @@ -10,7 +5,6 @@ ARG deb_version ARG deb_release=unstable ARG network=mainnet - ARG psql_version=13 # Golang version number used to detemine tarball name ARG GO_VERSION=1.19.11 @@ -70,12 +64,10 @@ RUN curl -s https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \ postgresql-"$psql_version" \ postgresql-client-"$psql_version" -# --- Install rosetta-cli - - # --- Golang install of a given GO_VERSION (add -v for spam output of each file from the go dist) RUN curl -s "https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz" | tar -xz -C /usr/lib/ +# --- Install rosetta-cli # The following commands install rosetta-cli with an updated rosetta-sdk-go dependency, which supports delegation transactions. # Upstream PR to rosetta-sdk-go: https://github.com/coinbase/rosetta-sdk-go/pull/457 # They can be replaced once there is a new release of rosetta-cli containing the above change by: @@ -88,17 +80,8 @@ RUN export GOBIN="$(pwd)/bin" \ && /usr/lib/go/bin/go mod tidy \ && /usr/lib/go/bin/go install -# Rosetta-demo artifacts / build dir removal: -# Make rosetta-crucial components and the generate_keypair tool -# && dune build --profile=${DUNE_PROFILE} \ -# && _build/default/src/app/runtime_genesis_ledger/runtime_genesis_ledger.exe \ -# --config-file src/app/rosetta/demo-config.json \ -# --genesis-dir ${HOME}/demo-genesis \ -# && mv _build/default/src/app $HOME/app \ -# && rm -rf _build - # --- Generate en_US.UTF-8 locale to allow use of O(1) Labs DB dumps -#RUN locale-gen en_US.UTF-8 +RUN locale-gen en_US.UTF-8 # --- Set up sample/demo wallets and configuration RUN mkdir -p --mode=700 ${MINA_CONFIG_DIR}/wallets/store/ \