From eab593394aea14c7eee4732c2325dc4fd36b9078 Mon Sep 17 00:00:00 2001 From: dkijania Date: Fri, 6 Oct 2023 20:52:18 +0200 Subject: [PATCH] New rosetta docker --- buildkite/scripts/build-artifact.sh | 3 +- .../4-production => Dockerfile-mina-rosetta} | 85 +++++++++++-------- dockerfiles/stages/3-builder | 72 ---------------- dockerfiles/stages/4-deb-builder | 58 ------------- scripts/rebuild-deb.sh | 3 +- scripts/release-docker.sh | 2 +- src/app/rosetta/rosetta_testnet_signatures.ml | 2 +- 7 files changed, 55 insertions(+), 170 deletions(-) rename dockerfiles/{stages/4-production => Dockerfile-mina-rosetta} (68%) mode change 100644 => 100755 delete mode 100644 dockerfiles/stages/3-builder delete mode 100644 dockerfiles/stages/4-deb-builder diff --git a/buildkite/scripts/build-artifact.sh b/buildkite/scripts/build-artifact.sh index adb98a79ff8..3af63a0e1dc 100755 --- a/buildkite/scripts/build-artifact.sh +++ b/buildkite/scripts/build-artifact.sh @@ -13,7 +13,7 @@ echo "--- Build libp2p_helper TODO: use the previously uploaded build artifact" make -C src/app/libp2p_helper MAINNET_TARGETS="" -[[ ${MINA_BUILD_MAINNET} ]] && MAINNET_TARGETS="src/app/cli/src/mina_mainnet_signatures.exe src/app/rosetta/rosetta_mainnet_signatures.exe " +[[ ${MINA_BUILD_MAINNET} ]] && MAINNET_TARGETS="src/app/cli/src/mina_mainnet_signatures.exe src/app/rosetta/rosetta_mainnet_signatures.exe src/app/rosetta/ocaml-signer/signer_mainnet_signatures.exe" echo "--- Build all major tagets required for packaging" echo "Building from Commit SHA: ${MINA_COMMIT_SHA1}" @@ -33,6 +33,7 @@ dune build "--profile=${DUNE_PROFILE}" \ src/app/swap_bad_balances/swap_bad_balances.exe \ src/app/zkapp_test_transaction/zkapp_test_transaction.exe \ src/app/rosetta/rosetta_testnet_signatures.exe \ + src/app/rosetta/ocaml-signer/signer_testnet_signatures.exe \ src/app/test_executive/test_executive.exe # 2>&1 | tee /tmp/buildocaml.log echo "--- Bundle all packages for Debian ${MINA_DEB_CODENAME}" diff --git a/dockerfiles/stages/4-production b/dockerfiles/Dockerfile-mina-rosetta old mode 100644 new mode 100755 similarity index 68% rename from dockerfiles/stages/4-production rename to dockerfiles/Dockerfile-mina-rosetta index 1e45c124f74..b4eaecbca06 --- a/dockerfiles/stages/4-production +++ b/dockerfiles/Dockerfile-mina-rosetta @@ -6,6 +6,10 @@ ARG image=ubuntu:focal FROM ${image} AS production ARG deb_codename=focal +ARG deb_version +ARG deb_release=unstable +ARG network=mainnet + ARG psql_version=13 @@ -18,6 +22,23 @@ ARG ROSETTA_CLI_VERSION=v0.10.1 ENV DEBIAN_FRONTEND noninteractive +# We do not install the below platform-specific dependencies, +# and instead assume that apt will install the proper deps based on the package definition. +# The packages are noted here just for clarity/documentation. +# Stretch-only Deps: +# libffi6 +# libprocps6 +# libjemalloc1 +# Buster-only Deps: +# libffi7 +# libprocps7 +# libjemalloc2 +# Bullseye/Focal-only Deps: +# libffi7 +# libprocps8 +# libjemalloc2 + + # --- Dependencies across many platforms RUN apt-get update --quiet --yes \ && apt-get upgrade --quiet --yes \ @@ -46,33 +67,6 @@ RUN curl -s https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \ postgresql-"$psql_version" \ postgresql-client-"$psql_version" -# --- Debian 11 "Bullseye" Dependencies -RUN test "$deb_codename" = "bullseye" \ - && apt-get update -y \ - && apt-get install -y \ - libjemalloc2 \ - libprocps8 \ - libffi7 \ - || exit 0 - -# --- Debian 10 "Buster" Dependencies -RUN test "$deb_codename" = "buster" \ - && apt-get update --quiet --yes \ - && apt-get install --quiet --yes --no-install-recommends \ - libjemalloc2 \ - libprocps7 \ - libffi6 \ - || exit 0 - -# --- Ubuntu 20.04 "Focal Fossa" Dependencies -RUN test "$deb_codename" = "focal" \ - && apt-get update --quiet --yes \ - && apt-get install --quiet --yes --no-install-recommends \ - libjemalloc2 \ - libprocps8 \ - libffi7 \ - || exit 0 - # --- Install rosetta-cli # The following commands install rosetta-cli with an updated rosetta-sdk-go dependency, which supports delegation transactions. @@ -89,6 +83,16 @@ RUN export GOBIN="$(pwd)/bin" \ && /usr/lib/go/bin/go mod tidy \ && /usr/lib/go/bin/go install + +# Rosetta-demo artifacts / build dir removal: +# Make rosetta-crucial components and the generate_keypair tool +# && dune build --profile=${DUNE_PROFILE} \ +# && _build/default/src/app/runtime_genesis_ledger/runtime_genesis_ledger.exe \ +# --config-file src/app/rosetta/demo-config.json \ +# --genesis-dir ${HOME}/demo-genesis \ +# && mv _build/default/src/app $HOME/app \ +# && rm -rf _build + # --- Generate en_US.UTF-8 locale to allow use of O(1) Labs DB dumps RUN locale-gen en_US.UTF-8 @@ -99,15 +103,24 @@ RUN mkdir -p --mode=700 ${MINA_CONFIG_DIR}/wallets/store/ \ > ${MINA_CONFIG_DIR}/wallets/store/${PK} \ && chmod go-rwx ${MINA_CONFIG_DIR}/wallets/store/${PK} -# --- Copy artifacts from the 3-builder stage -COPY --from=builder /home/opam/app/* /usr/local/bin/ -COPY --from=builder /home/opam/mina/src/app/rosetta/*.sh /rosetta/ -COPY --from=builder /home/opam/mina/src/app/rosetta/*.conf /rosetta/ -COPY --from=builder /home/opam/mina/src/app/rosetta/README.md /rosetta/ -COPY --from=builder /home/opam/mina/src/app/rosetta/rosetta-cli-config/*.json /rosetta/rosetta-cli-config/ -COPY --from=builder /home/opam/mina/src/app/rosetta/rosetta-cli-config/*.ros /rosetta/rosetta-cli-config/ -COPY --from=builder /home/opam/mina/src/app/archive/*.sql /archive/ -COPY --from=builder /home/opam/mina/genesis_ledgers /genesis_ledgers/ + +# Mina daemon package +# jemalloc is also installed automatically here to match the package dependencies for this $deb_codename +RUN echo "Building image with version $deb_version from repo $deb_release $deb_codename for network $network" \ + && echo "deb [trusted=yes] http://packages.o1test.net $deb_codename $deb_release" > /etc/apt/sources.list.d/o1.list \ + && apt-get update --quiet --yes \ + && apt-get install --quiet --yes --allow-downgrades "mina-$network=$deb_version" \ + && rm -rf /var/lib/apt/lists/* + + +# --- Copy artifacts +COPY src/app/rosetta/*.sh /rosetta/ +COPY src/app/rosetta/*.conf /rosetta/ +COPY src/app/rosetta/README.md /rosetta/ +COPY src/app/rosetta/rosetta-cli-config/*.json /rosetta/rosetta-cli-config/ +COPY src/app/rosetta/rosetta-cli-config/*.ros /rosetta/rosetta-cli-config/ +COPY src/app/archive/*.sql /archive/ +COPY genesis_ledgers /genesis_ledgers/ # --- Set up postgres USER postgres diff --git a/dockerfiles/stages/3-builder b/dockerfiles/stages/3-builder deleted file mode 100644 index 38b36f5d066..00000000000 --- a/dockerfiles/stages/3-builder +++ /dev/null @@ -1,72 +0,0 @@ -################################################################################################# -# The "builder" Stage -# - builds mina and any other binaries required to run rosetta -# - should not include any data related to joining a specific network, only the node software itself -################################################################################################# -FROM opam-deps AS builder - -# Use --build-arg "DUNE_PROFILE=dev" to build a dev image or for CI -ARG DUNE_PROFILE=devnet - -# branch to checkout on first clone (this will be the only available branch in the container) -# can also be a tagged release -ARG MINA_BRANCH=compatible - -# repo to checkout the branch from -ARG MINA_REPO=https://github.com/MinaProtocol/mina - -# location of repo used for pins and external package commits -ARG MINA_DIR=mina - -ENV PATH "$PATH:/usr/lib/go/bin:$HOME/.cargo/bin" - -# --- Shallow Clone the Mina repo -# Delete any previous clones of the mina repo and then clone into an empty directory -RUN cd $HOME && rm -rf $HOME/${MINA_DIR} \ - && git clone \ - -b "${MINA_BRANCH}" \ - --depth 1 \ - --shallow-submodules \ - --recurse-submodules \ - ${MINA_REPO} ${HOME}/${MINA_DIR} -# Set the working directory to the cloned mina repo -WORKDIR $HOME/${MINA_DIR} -# Create a directory for compiled outputs -RUN mkdir ${HOME}/app - -# --- Build libp2p_helper -RUN make libp2p_helper \ - && mv src/app/libp2p_helper/result/bin/libp2p_helper ${HOME}/app/libp2p_helper - -# --- Make rosetta-crucial components and the generate_keypair tool -RUN eval $(opam config env) \ - && dune build --profile=${DUNE_PROFILE} \ - src/app/cli/src/mina_testnet_signatures.exe \ - src/app/cli/src/mina_mainnet_signatures.exe \ - src/app/archive/archive.exe \ - src/app/archive_blocks/archive_blocks.exe \ - src/app/extract_blocks/extract_blocks.exe \ - src/app/missing_blocks_auditor/missing_blocks_auditor.exe \ - src/app/replayer/replayer.exe \ - src/app/rosetta/rosetta_testnet_signatures.exe \ - src/app/rosetta/rosetta_mainnet_signatures.exe \ - src/app/generate_keypair/generate_keypair.exe \ - src/app/validate_keypair/validate_keypair.exe \ - src/app/rosetta/ocaml-signer/signer.exe \ - && cp _build/default/src/app/archive_blocks/archive_blocks.exe $HOME/app/mina-archive-blocks \ - && cp _build/default/src/app/extract_blocks/extract_blocks.exe $HOME/app/mina-extract-blocks \ - && cp _build/default/src/app/missing_blocks_auditor/missing_blocks_auditor.exe $HOME/app/mina-missing-blocks-auditor \ - && cp _build/default/src/app/replayer/replayer.exe $HOME/app/mina-replayer \ - && mv _build/default/src/app/cli/src/mina_mainnet_signatures.exe $HOME/app/mina \ - && mv _build/default/src/app/cli/src/mina_testnet_signatures.exe $HOME/app/mina-dev \ - && mv _build/default/src/app/archive/archive.exe $HOME/app/mina-archive \ - && mv _build/default/src/app/rosetta/rosetta_mainnet_signatures.exe $HOME/app/mina-rosetta \ - && mv _build/default/src/app/rosetta/rosetta_testnet_signatures.exe $HOME/app/mina-rosetta-dev \ - && mv _build/default/src/app/generate_keypair/generate_keypair.exe $HOME/app/mina-generate-keypair \ - && mv _build/default/src/app/validate_keypair/validate_keypair.exe $HOME/app/mina-validate-keypair \ - && mv _build/default/src/app/rosetta/ocaml-signer/signer.exe $HOME/app/mina-ocaml-signer \ - && rm -rf _build - -# --- Clear go module caches to make the container smaller -RUN cd src/app/libp2p_helper/src/libp2p_helper \ - && go clean --cache --modcache --testcache -r diff --git a/dockerfiles/stages/4-deb-builder b/dockerfiles/stages/4-deb-builder deleted file mode 100644 index 38dbf10a2cf..00000000000 --- a/dockerfiles/stages/4-deb-builder +++ /dev/null @@ -1,58 +0,0 @@ -################################################################################################# -# The "deb-builder" Stage -# - builds mina and any other binaries required to run `make deb` -################################################################################################# -FROM toolchain AS deb-builder - -# Use --build-arg "DUNE_PROFILE=dev" to build a dev image or for CI -ARG DUNE_PROFILE=devnet - -# branch to checkout on first clone (this will be the only available branch in the container) -# can also be a tagged release -ARG MINA_BRANCH=compatible - -ARG MINA_COMMIT - -# repo to checkout the branch from -ARG MINA_REPO=https://github.com/MinaProtocol/mina - -# location of repo used for pins and external package commits -ARG MINA_DIR=mina - -ENV MINA_COMMIT_SHA1=${MINA_COMMIT} - -ENV PATH "$PATH:/usr/lib/go/bin:$HOME/.cargo/bin" - -# --- Build all major tagets required for packaging -RUN echo "--- Build all major tagets required for packaging" \ - && echo "Building from Commit SHA: ${MINA_COMMIT_SHA1}" \ - && eval $(opam config env) \ - && dune build --profile=${DUNE_PROFILE} \ - src/app/logproc/logproc.exe \ - src/app/runtime_genesis_ledger/runtime_genesis_ledger.exe \ - src/app/generate_keypair/generate_keypair.exe \ - src/app/validate_keypair/validate_keypair.exe \ - src/app/cli/src/mina.exe \ - src/app/cli/src/mina_testnet_signatures.exe \ - src/app/cli/src/mina_mainnet_signatures.exe \ - src/app/archive/archive.exe \ - src/app/archive/archive_testnet_signatures.exe \ - src/app/archive/archive_mainnet_signatures.exe \ - src/app/rosetta/rosetta.exe \ - src/app/rosetta/rosetta_mainnet_signatures.exe \ - src/app/rosetta/rosetta_testnet_signatures.exe \ - src/app/rosetta/ocaml-signer/signer.exe - -# --- MAKE DEB -RUN eval $(opam config env) \ - && echo "Building from Commit SHA: ${MINA_COMMIT_SHA1}" \ - && ./scripts/rebuild-deb.sh - -# Rosetta-demo artifacts / build dir removal: -# Make rosetta-crucial components and the generate_keypair tool -# && dune build --profile=${DUNE_PROFILE} \ -# && _build/default/src/app/runtime_genesis_ledger/runtime_genesis_ledger.exe \ -# --config-file src/app/rosetta/demo-config.json \ -# --genesis-dir ${HOME}/demo-genesis \ -# && mv _build/default/src/app $HOME/app \ -# && rm -rf _build diff --git a/scripts/rebuild-deb.sh b/scripts/rebuild-deb.sh index 770e1dcbf60..ea1e48894af 100755 --- a/scripts/rebuild-deb.sh +++ b/scripts/rebuild-deb.sh @@ -131,7 +131,8 @@ copy_common_daemon_configs() { # Copy signature-based Binaries (based on signature type $2 passed into the function) cp ./default/src/app/cli/src/mina_${2}_signatures.exe "${BUILDDIR}/usr/local/bin/mina" cp ./default/src/app/rosetta/rosetta_${2}_signatures.exe "${BUILDDIR}/usr/local/bin/mina-rosetta" - + cp ./default/src/app/rosetta/ocaml-signer/signer_${2}_signatures.exe "${BUILDDIR}/usr/local/bin/mina-rosetta-ocaml-signer" + # Copy over Build Configs (based on $2) mkdir -p "${BUILDDIR}/etc/coda/build_config" # Use parameter expansion to either return "mainnet.mlh" or "devnet.mlh" diff --git a/scripts/release-docker.sh b/scripts/release-docker.sh index 4407af6656f..cbf5b26289c 100755 --- a/scripts/release-docker.sh +++ b/scripts/release-docker.sh @@ -95,7 +95,7 @@ mina-batch-txn) VERSION="${VERSION}-${NETWORK##*=}" ;; mina-rosetta) - DOCKERFILE_PATH="dockerfiles/stages/1-build-deps dockerfiles/stages/2-opam-deps dockerfiles/stages/3-builder dockerfiles/stages/4-production" + DOCKERFILE_PATH="dockerfiles/Dockerfile-mina-rosetta" ;; mina-zkapp-test-transaction) DOCKERFILE_PATH="dockerfiles/Dockerfile-zkapp-test-transaction" diff --git a/src/app/rosetta/rosetta_testnet_signatures.ml b/src/app/rosetta/rosetta_testnet_signatures.ml index ca32b93398a..b7a558d7145 100644 --- a/src/app/rosetta/rosetta_testnet_signatures.ml +++ b/src/app/rosetta/rosetta_testnet_signatures.ml @@ -3,4 +3,4 @@ open Async let () = Command.run - (Command.async ~summary:"Run Rosetta process on top of Coda" command) + (Command.async ~summary:"Run Rosetta process on top of Mina" command)