diff --git a/README.md b/README.md index b848104e1d8..b7c73df3815 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,16 @@ ### Build status -| Develop | Berkeley | Compatible | -| ------- | -------- | ---------- | -| [![Build status - develop](https://badge.buildkite.com/0c47452f3ea619d3217d388e0de522b218db28c3e161887a9a.svg?branch=develop)](https://buildkite.com/o-1-labs-2/mina-end-to-end-nightlies) | [![Build status - berkeley](https://badge.buildkite.com/0c47452f3ea619d3217d388e0de522b218db28c3e161887a9a.svg?branch=berkeley)](https://buildkite.com/o-1-labs-2/mina-end-to-end-nightlies) | [![Build status - compatible](https://badge.buildkite.com/0c47452f3ea619d3217d388e0de522b218db28c3e161887a9a.svg?branch=compatible)](https://buildkite.com/o-1-labs-2/mina-end-to-end-nightlies) +| Develop | Compatible | Master | +| ------- | ---------- | ---------- | +| [![Build status - develop](https://badge.buildkite.com/0c47452f3ea619d3217d388e0de522b218db28c3e161887a9a.svg?branch=develop)](https://buildkite.com/o-1-labs-2/mina-end-to-end-nightlies) | [![Build status - compatible](https://badge.buildkite.com/0c47452f3ea619d3217d388e0de522b218db28c3e161887a9a.svg?branch=compatible)](https://buildkite.com/o-1-labs-2/mina-end-to-end-nightlies) | [![Build status - master](https://badge.buildkite.com/0c47452f3ea619d3217d388e0de522b218db28c3e161887a9a.svg?branch=master)](https://buildkite.com/o-1-labs-2/mina-end-to-end-nightlies) + +
- Mina logo + Mina logo -# Mina +
Mina is the first cryptocurrency with a lightweight, constant-sized blockchain. This is the main source code repository for the Mina project and contains code for the OCaml protocol implementation, the [Mina Protocol website](https://minaprotocol.com), and wallet. Enjoy! diff --git a/buildkite/scripts/build-artifact.sh b/buildkite/scripts/build-artifact.sh index 263a258ef3c..72dc393fe59 100755 --- a/buildkite/scripts/build-artifact.sh +++ b/buildkite/scripts/build-artifact.sh @@ -24,7 +24,7 @@ make -C src/app/libp2p_helper MAINNET_TARGETS="" [[ ${MINA_BUILD_MAINNET} ]] && MAINNET_TARGETS="src/app/cli/src/mina_mainnet_signatures.exe src/app/rosetta/rosetta_mainnet_signatures.exe src/app/rosetta/ocaml-signer/signer_mainnet_signatures.exe" -echo "--- Build all major tagets required for packaging" +echo "--- Build all major targets required for packaging" echo "Building from Commit SHA: ${MINA_COMMIT_SHA1}" echo "Rust Version: $(rustc --version)" dune build "--profile=${DUNE_PROFILE}" $INSTRUMENTED_PARAM \ diff --git a/buildkite/scripts/build-release.sh b/buildkite/scripts/build-release.sh index cf3727818c4..fdd6440cd5c 100755 --- a/buildkite/scripts/build-release.sh +++ b/buildkite/scripts/build-release.sh @@ -9,7 +9,7 @@ source ~/.profile ./buildkite/scripts/build-artifact.sh echo "--- Bundle all packages for Debian ${MINA_DEB_CODENAME}" -echo " Includes mina daemon, archive-node, rosetta, generate keypair for berkeley" +echo " Includes mina daemon, archive-node, rosetta, generate keypair for devnet" [[ ${MINA_BUILD_MAINNET} ]] && echo " MINA_BUILD_MAINNET is true so this includes the mainnet and devnet packages for mina-daemon as well" diff --git a/buildkite/scripts/check-compatibility.sh b/buildkite/scripts/check-compatibility.sh index c638ec813c4..43f5dd4073b 100755 --- a/buildkite/scripts/check-compatibility.sh +++ b/buildkite/scripts/check-compatibility.sh @@ -11,7 +11,7 @@ function get_shas { function image_tag { SHA=$1 - IMAGE_TAG="$SHA-bullseye-berkeley" + IMAGE_TAG="$SHA-bullseye-devnet" } function download-docker { diff --git a/buildkite/scripts/connect-to-berkeley.sh b/buildkite/scripts/connect-to-berkeley.sh deleted file mode 100755 index 1b2aa7cfe82..00000000000 --- a/buildkite/scripts/connect-to-berkeley.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/bin/bash - -set -eo pipefail - -case "$BUILDKITE_PULL_REQUEST_BASE_BRANCH" in - rampup|berkeley|release/2.0.0|develop) - ;; - *) - echo "Not pulling against rampup, not running the connect test" - exit 0 ;; -esac - -# Don't prompt for answers during apt-get install -export DEBIAN_FRONTEND=noninteractive - -apt-get update -apt-get install -y git apt-transport-https ca-certificates tzdata curl - -TESTNET_VERSION_NAME="berkeley" -TESTNET_NAME="testworld-2-0" - -git config --global --add safe.directory /workdir - -source buildkite/scripts/export-git-env-vars.sh - -source buildkite/scripts/debian/install.sh "mina-${TESTNET_VERSION_NAME}" - -# Remove lockfile if present -rm ~/.mina-config/.mina-lock ||: - -mkdir -p /root/libp2p-keys/ -# Pre-generated random password for this quick test -export MINA_LIBP2P_PASS=eithohShieshichoh8uaJ5iefo1reiRudaekohG7AeCeib4XuneDet2uGhu7lahf -mina libp2p generate-keypair --privkey-path /root/libp2p-keys/key -# Set permissions on the keypair so the daemon doesn't complain -chmod -R 0700 /root/libp2p-keys/ - -# Restart in the background -mina daemon \ - --peer-list-url "https://storage.googleapis.com/seed-lists/${TESTNET_NAME}_seeds.txt" \ - --libp2p-keypair "/root/libp2p-keys/key" \ -& # -background - -# Attempt to connect to the GraphQL client every 30s for up to 12 minutes -num_status_retries=24 -for ((i=1;i<=$num_status_retries;i++)); do - sleep 30s - set +e - mina client status - status_exit_code=$? - set -e - if [ $status_exit_code -eq 0 ]; then - break - elif [ $i -eq $num_status_retries ]; then - exit $status_exit_code - fi -done - -# Check that the daemon has connected to peers and is still up after 2 mins -sleep 2m -mina client status -if [ $(mina advanced get-peers | wc -l) -gt 0 ]; then - echo "Found some peers" -else - echo "No peers found" - exit 1 -fi - diff --git a/buildkite/scripts/connect-to-testnet.sh b/buildkite/scripts/connect-to-testnet.sh index 3cfd7a4aa7b..cba27c93759 100755 --- a/buildkite/scripts/connect-to-testnet.sh +++ b/buildkite/scripts/connect-to-testnet.sh @@ -7,53 +7,46 @@ if [[ $# -ne 3 ]]; then exit 1 fi -TESTNET_VERSION_NAME="berkeley" TESTNET_NAME=$1 WAIT_BETWEEN_POLLING_GRAPHQL=$2 WAIT_AFTER_FINAL_CHECK=$3 -case "$BUILDKITE_PULL_REQUEST_BASE_BRANCH" in - rampup|berkeley|release/2.0.0|develop) - ;; - *) - echo "Not pulling against rampup, not running the connect test" - exit 0 ;; -esac - # Don't prompt for answers during apt-get install export DEBIAN_FRONTEND=noninteractive -apt-get update -apt-get install -y git apt-transport-https ca-certificates tzdata curl +sudo apt-get update +sudo apt-get install -y git apt-transport-https ca-certificates tzdata curl libwww-perl jq git config --global --add safe.directory /workdir source buildkite/scripts/export-git-env-vars.sh -source buildkite/scripts/debian/install.sh "mina-${TESTNET_VERSION_NAME}" +source buildkite/scripts/debian/install.sh "mina-${TESTNET_NAME}" 1 # Remove lockfile if present -rm ~/.mina-config/.mina-lock ||: +sudo rm ~/.mina-config/.mina-lock ||: + +sudo mkdir -p /root/libp2p-keys/ -mkdir -p /root/libp2p-keys/ -# Pre-generated random password for this quick test -export MINA_LIBP2P_PASS=eithohShieshichoh8uaJ5iefo1reiRudaekohG7AeCeib4XuneDet2uGhu7lahf -mina libp2p generate-keypair --privkey-path /root/libp2p-keys/key # Set permissions on the keypair so the daemon doesn't complain -chmod -R 0700 /root/libp2p-keys/ +sudo chmod -R 0700 /root/libp2p-keys/ +# Pre-generated random password for this quick test +sudo MINA_LIBP2P_PASS=eithohShieshichoh8uaJ5iefo1reiRudaekohG7AeCeib4XuneDet2uGhu7lahf mina libp2p generate-keypair --privkey-path /root/libp2p-keys/key # Restart in the background -mina daemon \ - --peer-list-url "https://storage.googleapis.com/seed-lists/${TESTNET_NAME}_seeds.txt" \ - --libp2p-keypair "/root/libp2p-keys/key" \ -& # -background +sudo MINA_LIBP2P_PASS=eithohShieshichoh8uaJ5iefo1reiRudaekohG7AeCeib4XuneDet2uGhu7lahf \ + TESTNET_NAME=$TESTNET_NAME \ + bash -c "mina daemon \ + --peer-list-url \"https://storage.googleapis.com/seed-lists/${TESTNET_NAME}_seeds.txt\" \ + --libp2p-keypair \"/root/libp2p-keys/key\" \ + --seed &" # -background # Attempt to connect to the GraphQL client every 10s for up to 8 minutes num_status_retries=24 for ((i=1;i<=$num_status_retries;i++)); do sleep $WAIT_BETWEEN_POLLING_GRAPHQL set +e - mina client status + sudo mina client status status_exit_code=$? set -e if [ $status_exit_code -eq 0 ]; then @@ -63,13 +56,18 @@ for ((i=1;i<=$num_status_retries;i++)); do fi done -# Check that the daemon has connected to peers and is still up after 2 mins -sleep $WAIT_AFTER_FINAL_CHECK -mina client status -if [ $(mina advanced get-peers | wc -l) -gt 0 ]; then - echo "Found some peers" -else - echo "No peers found" - exit 1 -fi +peer_retries=10 +for ((i=1;i<=$peer_retries;i++)); do + peer_count=$(sudo mina advanced get-peers | wc -l) + sudo mina client status + + if [ "$peer_count" -gt 0 ]; then + echo "Found some peers" + exit 0; + else + echo "No peers found" + fi + sleep $WAIT_AFTER_FINAL_CHECK +done +exit 1; \ No newline at end of file diff --git a/buildkite/scripts/debian/install.sh b/buildkite/scripts/debian/install.sh index 93313f4a2dd..bd9a863f8ef 100755 --- a/buildkite/scripts/debian/install.sh +++ b/buildkite/scripts/debian/install.sh @@ -34,7 +34,7 @@ else debs=(${DEBS//,/ }) for i in "${debs[@]}"; do case $i in - mina-berkeley|mina-devnet|mina-mainnet|mina-berkeley-lightnet) + mina-devnet|mina-mainnet|mina-devnet-lightnet) # Downaload mina-logproc too source ./buildkite/scripts/download-artifact-from-cache.sh "mina-logproc*" $MINA_DEB_CODENAME/_build "" $LOCAL_DEB_FOLDER ;; diff --git a/buildkite/scripts/dump-mina-type-shapes.sh b/buildkite/scripts/dump-mina-type-shapes.sh index 57d3c2b2302..3855157e017 100755 --- a/buildkite/scripts/dump-mina-type-shapes.sh +++ b/buildkite/scripts/dump-mina-type-shapes.sh @@ -8,7 +8,7 @@ export DEBIAN_FRONTEND=noninteractive sudo apt-get update sudo apt-get install -y git apt-transport-https ca-certificates tzdata curl -TESTNET_NAME="berkeley" +TESTNET_NAME="devnet" git config --global --add safe.directory /workdir source buildkite/scripts/export-git-env-vars.sh diff --git a/buildkite/scripts/rosetta-integration-tests.sh b/buildkite/scripts/rosetta-integration-tests.sh index c3fb9dd378a..e091ca39269 100755 --- a/buildkite/scripts/rosetta-integration-tests.sh +++ b/buildkite/scripts/rosetta-integration-tests.sh @@ -250,10 +250,12 @@ if [[ "$MODE" == "full" ]]; then send_zkapp_transactions & fi -next_block_time=$(mina client status --json | jq '.next_block_production.timing[1].time' | tr -d '"') curr_time=$(date +%s%N | cut -b1-13) -sleep_time=$((($next_block_time - $curr_time) / 1000)) +mina client status --json + +next_block_time=$(mina client status --json | jq '.next_block_production.timing[1]' | tr -d '"') curr_time=$(date +%s%N | cut -b1-13) +sleep_time=$((($next_block_time - $curr_time) / 10000000)) echo "Sleeping for ${sleep_time}s until next block is created..." -sleep ${sleep_time} +sleep $sleep_time # Mina Rosetta Checks (spec construction data perf) echo "============ ROSETTA CLI: VALIDATE CONF FILE ${ROSETTA_CONFIGURATION_FILE} ==============" diff --git a/buildkite/scripts/run-snark-transaction-profiler.sh b/buildkite/scripts/run-snark-transaction-profiler.sh index 802cd730632..70c20a92167 100755 --- a/buildkite/scripts/run-snark-transaction-profiler.sh +++ b/buildkite/scripts/run-snark-transaction-profiler.sh @@ -8,7 +8,7 @@ export DEBIAN_FRONTEND=noninteractive sudo apt-get update sudo apt-get install -y git apt-transport-https ca-certificates tzdata curl python3 -TESTNET_NAME="berkeley" +TESTNET_NAME="devnet" git config --global --add safe.directory /workdir source buildkite/scripts/export-git-env-vars.sh diff --git a/buildkite/scripts/run-test-executive-cloud.sh b/buildkite/scripts/run-test-executive-cloud.sh index 884b8222cbc..8a3df280401 100755 --- a/buildkite/scripts/run-test-executive-cloud.sh +++ b/buildkite/scripts/run-test-executive-cloud.sh @@ -2,7 +2,7 @@ set -o pipefail -x TEST_NAME="$1" -MINA_IMAGE="gcr.io/o1labs-192920/mina-daemon:$MINA_DOCKER_TAG-berkeley" +MINA_IMAGE="gcr.io/o1labs-192920/mina-daemon:$MINA_DOCKER_TAG-devnet" ARCHIVE_IMAGE="gcr.io/o1labs-192920/mina-archive:$MINA_DOCKER_TAG" if [[ "${TEST_NAME:0:15}" == "block-prod-prio" ]] && [[ "$RUN_OPT_TESTS" == "" ]]; then diff --git a/buildkite/scripts/run-test-executive-local.sh b/buildkite/scripts/run-test-executive-local.sh index 09c37c2b206..8512b93df0f 100755 --- a/buildkite/scripts/run-test-executive-local.sh +++ b/buildkite/scripts/run-test-executive-local.sh @@ -26,7 +26,7 @@ cleanup TEST_NAME="$1" -MINA_IMAGE="gcr.io/o1labs-192920/mina-daemon:$MINA_DOCKER_TAG-berkeley" +MINA_IMAGE="gcr.io/o1labs-192920/mina-daemon:$MINA_DOCKER_TAG-devnet" ARCHIVE_IMAGE="gcr.io/o1labs-192920/mina-archive:$MINA_DOCKER_TAG" if [[ "${TEST_NAME:0:15}" == "block-prod-prio" ]] && [[ "$RUN_OPT_TESTS" == "" ]]; then @@ -42,7 +42,7 @@ rm -f /etc/apt/sources.list.d/hashicorp.list apt-get update apt-get install -y git apt-transport-https ca-certificates tzdata curl -TESTNET_NAME="berkeley" +TESTNET_NAME="devnet" git config --global --add safe.directory /workdir diff --git a/buildkite/scripts/run_promote_build_job.sh b/buildkite/scripts/run_promote_build_job.sh index 3de86fc13de..68e827f9392 100755 --- a/buildkite/scripts/run_promote_build_job.sh +++ b/buildkite/scripts/run_promote_build_job.sh @@ -61,7 +61,7 @@ DHALL_DEBIANS="([] : List $DEBIAN_DHALL_DEF.Type)" if [[ -n "$DEBIANS" ]]; then if [[ -z "$CODENAMES" ]]; then usage "Codenames is not set!"; exit 1; fi; if [[ -z "$PROFILE" ]]; then PROFILE="Standard"; fi; - if [[ -z "$NETWORK" ]]; then NETWORK="Berkeley"; fi; + if [[ -z "$NETWORK" ]]; then NETWORK="Devnet"; fi; if [[ -z "$REMOVE_PROFILE_FROM_NAME" ]]; then REMOVE_PROFILE_FROM_NAME=0; fi; if [[ -z "$PUBLISH" ]]; then PUBLISH=0; fi; if [[ -z "$FROM_CHANNEL" ]]; then FROM_CHANNEL="Unstable"; fi; diff --git a/buildkite/scripts/run_verify_promoted_build_job.sh b/buildkite/scripts/run_verify_promoted_build_job.sh index da2a3789d3f..13e1f5f62cc 100755 --- a/buildkite/scripts/run_verify_promoted_build_job.sh +++ b/buildkite/scripts/run_verify_promoted_build_job.sh @@ -59,7 +59,7 @@ DHALL_DEBIANS="([] : List $DEBIAN_DHALL_DEF.Type)" if [[ -n "$DEBIANS" ]]; then if [[ -z "$CODENAMES" ]]; then usage "Codenames is not set!"; exit 1; fi; if [[ -z "$PROFILE" ]]; then PROFILE="Standard"; fi; - if [[ -z "$NETWORK" ]]; then NETWORK="Berkeley"; fi; + if [[ -z "$NETWORK" ]]; then NETWORK="Devnet"; fi; if [[ -z "$REMOVE_PROFILE_FROM_NAME" ]]; then REMOVE_PROFILE_FROM_NAME=0; fi; if [[ -z "$PUBLISH" ]]; then PUBLISH=0; fi; if [[ -z "$TO_CHANNEL" ]]; then TO_CHANNEL="Unstable"; fi; diff --git a/buildkite/scripts/single-node-tests.sh b/buildkite/scripts/single-node-tests.sh index 93b224c5ba8..dcd6752ae23 100755 --- a/buildkite/scripts/single-node-tests.sh +++ b/buildkite/scripts/single-node-tests.sh @@ -12,7 +12,7 @@ git config --global --add safe.directory /workdir source buildkite/scripts/export-git-env-vars.sh -source buildkite/scripts/debian/install.sh "mina-test-suite,mina-berkeley-lightnet" 1 +source buildkite/scripts/debian/install.sh "mina-test-suite,mina-devnet-lightnet" 1 export MINA_LIBP2P_PASS="naughty blue worm" export MINA_PRIVKEY_PASS="naughty blue worm" diff --git a/buildkite/scripts/unit-test.sh b/buildkite/scripts/unit-test.sh index de885193086..b38ad7d94a8 100755 --- a/buildkite/scripts/unit-test.sh +++ b/buildkite/scripts/unit-test.sh @@ -21,10 +21,10 @@ export LIBP2P_NIXLESS=1 PATH=/usr/lib/go/bin:$PATH GO=/usr/lib/go/bin/go time make build echo "--- Build all targets" -dune build "${path}" --profile="${profile}" -j16 +dune build "${path}" --profile="${profile}" -j8 echo "--- Check for changes to verification keys" -time dune runtest "src/app/print_blockchain_snark_vk" --profile="${profile}" -j16 +time dune runtest "src/app/print_blockchain_snark_vk" --profile="${profile}" -j8 # Turn on the proof-cache assertion, so that CI will fail if the proofs need to # be updated. @@ -35,8 +35,8 @@ export ERROR_ON_PROOF=true # skip running all of the tests that have already succeeded, since dune will # only retry those tests that failed. echo "--- Run unit tests" -time dune runtest "${path}" --profile="${profile}" -j16 || \ +time dune runtest "${path}" --profile="${profile}" -j8 || \ (./scripts/link-coredumps.sh && \ echo "--- Retrying failed unit tests" && \ - time dune runtest "${path}" --profile="${profile}" -j16 || \ + time dune runtest "${path}" --profile="${profile}" -j8 || \ (./scripts/link-coredumps.sh && false)) diff --git a/buildkite/scripts/version-linter.sh b/buildkite/scripts/version-linter.sh index fb1ea8c9d89..8af413c2436 100755 --- a/buildkite/scripts/version-linter.sh +++ b/buildkite/scripts/version-linter.sh @@ -7,7 +7,7 @@ if [[ $# -ne 1 ]]; then exit 1 fi -TESTNET_NAME="${TESTNET_NAME:-berkeley}" +TESTNET_NAME="${TESTNET_NAME:-devnet}" # Don't prompt for answers during apt-get install export DEBIAN_FRONTEND=noninteractive diff --git a/buildkite/src/Command/ConnectToTestnet.dhall b/buildkite/src/Command/ConnectToTestnet.dhall index ba4bba1b974..91bf6ee8bea 100644 --- a/buildkite/src/Command/ConnectToTestnet.dhall +++ b/buildkite/src/Command/ConnectToTestnet.dhall @@ -4,9 +4,9 @@ let Command = ./Base.dhall let Size = ./Size.dhall -let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type +let RunInToolchain = ./RunInToolchain.dhall -let Cmd = ../Lib/Cmds.dhall +let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type in { step = \(dependsOn : List Command.TaggedKey.Type) @@ -17,12 +17,9 @@ in { step = -> Command.build Command.Config::{ , commands = - [ Cmd.runInDocker - Cmd.Docker::{ - , image = (../Constants/ContainerImages.dhall).ubuntu2004 - } + RunInToolchain.runInToolchain + ([] : List Text) "./buildkite/scripts/connect-to-testnet.sh ${testnet} ${wait_between_graphql_poll} ${wait_before_final_check}" - ] , label = "Connect to ${testnet}" , soft_fail = Some soft_fail , key = "connect-to-${testnet}" diff --git a/buildkite/src/Command/DockerImage.dhall b/buildkite/src/Command/DockerImage.dhall index 1199d461332..917a50fb621 100644 --- a/buildkite/src/Command/DockerImage.dhall +++ b/buildkite/src/Command/DockerImage.dhall @@ -10,8 +10,6 @@ let Size = ./Size.dhall let Profiles = ../Constants/Profiles.dhall -let Artifacts = ../Constants/Artifacts.dhall - let BuildFlags = ../Constants/BuildFlags.dhall let Cmd = ../Lib/Cmds.dhall @@ -20,57 +18,81 @@ let DockerLogin = ../Command/DockerLogin/Type.dhall let DebianRepo = ../Constants/DebianRepo.dhall +let DebianVersions = ../Constants/DebianVersions.dhall + +let Network = ../Constants/Network.dhall + +let Artifacts = ../Constants/Artifacts.dhall + let ReleaseSpec = { Type = { deps : List Command.TaggedKey.Type , network : Text - , service : Text + , service : Artifacts.Type , version : Text , branch : Text , repo : Text , no_cache : Bool - , deb_codename : Text + , deb_codename : DebianVersions.DebVersion , deb_release : Text , deb_version : Text , deb_profile : Profiles.Type , deb_repo : DebianRepo.Type , build_flags : BuildFlags.Type - , step_key : Text + , step_key_suffix : Text , if : Optional B/If } , default = { deps = [] : List Command.TaggedKey.Type - , network = "devnet" + , network = "${Network.lowerName Network.Type.Devnet}" , version = "\\\${MINA_DOCKER_TAG}" - , service = Artifacts.dockerName Artifacts.Type.Daemon + , service = Artifacts.Type.Daemon , branch = "\\\${BUILDKITE_BRANCH}" , repo = "\\\${BUILDKITE_REPO}" - , deb_codename = "bullseye" + , deb_codename = DebianVersions.DebVersion.Bullseye , deb_release = "\\\${MINA_DEB_RELEASE}" , deb_version = "\\\${MINA_DEB_VERSION}" , deb_profile = Profiles.Type.Standard , build_flags = BuildFlags.Type.None , deb_repo = DebianRepo.Type.PackagesO1Test , no_cache = False - , step_key = "daemon-standard-docker-image" + , step_key_suffix = "-docker-image" , if = None B/If } } +let stepKey = + \(spec : ReleaseSpec.Type) + -> "${Artifacts.lowerName + spec.service}${Profiles.toLabelSegment + spec.deb_profile}${BuildFlags.toLabelSegment + spec.build_flags}${spec.step_key_suffix}" + +let stepLabel = + \(spec : ReleaseSpec.Type) + -> "Docker: ${Artifacts.capitalName + spec.service} ${spec.network} ${DebianVersions.capitalName + spec.deb_codename} ${Profiles.toSuffixUppercase + spec.deb_profile} ${BuildFlags.toSuffixUppercase + spec.build_flags}" + let generateStep = \(spec : ReleaseSpec.Type) - -> let exportMinaDebCmd = "export MINA_DEB_CODENAME=${spec.deb_codename}" + -> let exportMinaDebCmd = + "export MINA_DEB_CODENAME=${DebianVersions.lowerName + spec.deb_codename}" let maybeCacheOption = if spec.no_cache then "--no-cache" else "" let buildDockerCmd = "./scripts/docker/build.sh" - ++ " --service ${spec.service}" + ++ " --service ${Artifacts.dockerName spec.service}" ++ " --network ${spec.network}" ++ " --version ${spec.version}" ++ " --branch ${spec.branch}" ++ " ${maybeCacheOption} " - ++ " --deb-codename ${spec.deb_codename}" + ++ " --deb-codename ${DebianVersions.lowerName + spec.deb_codename}" ++ " --deb-repo ${DebianRepo.address spec.deb_repo}" ++ " --deb-release ${spec.deb_release}" ++ " --deb-version ${spec.deb_version}" @@ -81,10 +103,11 @@ let generateStep = let releaseDockerCmd = "./scripts/docker/release.sh" - ++ " --service ${spec.service}" + ++ " --service ${Artifacts.dockerName spec.service}" ++ " --version ${spec.version}" ++ " --network ${spec.network}" - ++ " --deb-codename ${spec.deb_codename}" + ++ " --deb-codename ${DebianVersions.lowerName + spec.deb_codename}" ++ " --deb-version ${spec.deb_version}" ++ " --deb-profile ${Profiles.lowerName spec.deb_profile}" ++ " --deb-build-flags ${BuildFlags.lowerName @@ -121,12 +144,16 @@ let generateStep = in Command.build Command.Config::{ , commands = commands - , label = "Docker: ${spec.step_key}" - , key = spec.step_key + , label = "${stepLabel spec}" + , key = "${stepKey spec}" , target = Size.XLarge , docker_login = Some DockerLogin::{=} , depends_on = spec.deps , if = spec.if } -in { generateStep = generateStep, ReleaseSpec = ReleaseSpec } +in { generateStep = generateStep + , ReleaseSpec = ReleaseSpec + , stepKey = stepKey + , stepLabel = stepLabel + } diff --git a/buildkite/src/Command/HardforkPackageGeneration.dhall b/buildkite/src/Command/HardforkPackageGeneration.dhall index 183826d81b4..a8b9edf84b7 100644 --- a/buildkite/src/Command/HardforkPackageGeneration.dhall +++ b/buildkite/src/Command/HardforkPackageGeneration.dhall @@ -98,6 +98,17 @@ let pipeline "gcr.io/o1labs-192920/mina-daemon:\${BUILDKITE_COMMIT:0:7}-${DebianVersions.lowerName debVersion}-${network_name}" + let dockerSpec = + DockerImage.ReleaseSpec::{ + , deps = + [ { name = pipelineName, key = generateLedgersJobKey } ] + , service = Artifacts.Type.Daemon + , network = network_name + , deb_codename = debVersion + , deb_profile = profile + , deb_repo = DebianRepo.Type.Local + } + in Pipeline.Config::{ , spec = JobSpec::{ , dirtyWhen = [ S.everything ] @@ -155,20 +166,7 @@ let pipeline , key = "publish-hardfork-deb-pkg" , target = Size.Small } - , DockerImage.generateStep - DockerImage.ReleaseSpec::{ - , deps = - [ { name = pipelineName, key = generateLedgersJobKey } ] - , service = Artifacts.dockerName Artifacts.Type.Daemon - , network = network_name - , deb_codename = "${DebianVersions.lowerName debVersion}" - , deb_profile = profile - , deb_repo = DebianRepo.Type.Local - , step_key = - "daemon-berkeley-${DebianVersions.lowerName - debVersion}${Profiles.toLabelSegment - profile}-docker-image" - } + , DockerImage.generateStep dockerSpec , Command.build Command.Config::{ , commands = @@ -182,10 +180,7 @@ let pipeline , target = Size.XLarge , depends_on = [ { name = pipelineName - , key = - "daemon-berkeley-${DebianVersions.lowerName - debVersion}${Profiles.toLabelSegment - profile}-docker-image" + , key = DockerImage.stepKey dockerSpec } ] , if = None B/If @@ -202,10 +197,7 @@ let pipeline , target = Size.XLarge , depends_on = [ { name = pipelineName - , key = - "daemon-berkeley-${DebianVersions.lowerName - debVersion}${Profiles.toLabelSegment - profile}-docker-image" + , key = DockerImage.stepKey dockerSpec } ] , if = None B/If @@ -214,28 +206,20 @@ let pipeline DockerImage.ReleaseSpec::{ , deps = [ { name = pipelineName, key = generateLedgersJobKey } ] - , service = Artifacts.dockerName Artifacts.Type.Archive + , service = Artifacts.Type.Archive , network = network_name - , deb_codename = "${DebianVersions.lowerName debVersion}" + , deb_codename = debVersion , deb_profile = profile , deb_repo = DebianRepo.Type.Local - , step_key = - "archive-${DebianVersions.lowerName - debVersion}${Profiles.toLabelSegment - profile}-docker-image" } , DockerImage.generateStep DockerImage.ReleaseSpec::{ , deps = [ { name = pipelineName, key = generateLedgersJobKey } ] - , service = Artifacts.dockerName Artifacts.Type.Rosetta + , service = Artifacts.Type.Rosetta , network = network_name , deb_repo = DebianRepo.Type.Local - , deb_codename = "${DebianVersions.lowerName debVersion}" - , step_key = - "rosetta-${DebianVersions.lowerName - debVersion}${Profiles.toLabelSegment - profile}-docker-image" + , deb_codename = debVersion } ] } diff --git a/buildkite/src/Command/MinaArtifact.dhall b/buildkite/src/Command/MinaArtifact.dhall index 28793fb3d22..1af0da3f865 100644 --- a/buildkite/src/Command/MinaArtifact.dhall +++ b/buildkite/src/Command/MinaArtifact.dhall @@ -42,7 +42,7 @@ let MinaBuildSpec = , artifacts : List Artifacts.Type , debVersion : DebianVersions.DebVersion , profile : Profiles.Type - , networks : List Network.Type + , network : Network.Type , buildFlags : BuildFlags.Type , toolchainSelectMode : Toolchain.SelectionMode , mode : PipelineMode.Type @@ -55,7 +55,7 @@ let MinaBuildSpec = , debVersion = DebianVersions.DebVersion.Bullseye , profile = Profiles.Type.Standard , buildFlags = BuildFlags.Type.None - , networks = [ Network.Type.Berkeley ] + , network = Network.Type.Devnet , toolchainSelectMode = Toolchain.SelectionMode.ByDebian , mode = PipelineMode.Type.PullRequest , tags = [ PipelineTag.Type.Long, PipelineTag.Type.Release ] @@ -63,6 +63,30 @@ let MinaBuildSpec = } } +let labelSuffix + : MinaBuildSpec.Type -> Text + = \(spec : MinaBuildSpec.Type) + -> "${DebianVersions.capitalName + spec.debVersion} ${Network.capitalName + spec.network} ${Profiles.toSuffixUppercase + spec.profile} ${BuildFlags.toSuffixUppercase + spec.buildFlags}" + +let nameSuffix + : MinaBuildSpec.Type -> Text + = \(spec : MinaBuildSpec.Type) + -> "${DebianVersions.capitalName + spec.debVersion}${Network.capitalName + spec.network}${Profiles.toSuffixUppercase + spec.profile}${BuildFlags.toSuffixUppercase + spec.buildFlags}" + +let keySuffix + : MinaBuildSpec.Type -> Text + = \(spec : MinaBuildSpec.Type) + -> "${Profiles.toLabelSegment spec.profile}${BuildFlags.toLabelSegment + spec.buildFlags}" + let build_artifacts : MinaBuildSpec.Type -> Command.Type = \(spec : MinaBuildSpec.Type) @@ -79,22 +103,18 @@ let build_artifacts , "MINA_COMMIT_SHA1=\$BUILDKITE_COMMIT" , "MINA_DEB_CODENAME=${DebianVersions.lowerName spec.debVersion}" - , Network.foldMinaBuildMainnetEnv spec.networks + , Network.buildMainnetEnv spec.network ] # BuildFlags.buildEnvs spec.buildFlags ) "./buildkite/scripts/build-release.sh ${Artifacts.toDebianNames spec.artifacts - spec.networks}" + spec.network}" # [ Cmd.run "./buildkite/scripts/debian/upload-to-gs.sh ${DebianVersions.lowerName spec.debVersion}" ] - , label = - "Build Mina for ${DebianVersions.capitalName - spec.debVersion} ${Profiles.toSuffixUppercase - spec.profile} ${BuildFlags.toSuffixUppercase - spec.buildFlags}" + , label = "Debian: Build ${labelSuffix spec}" , key = "build-deb-pkg" , target = Size.XLarge , retries = @@ -105,36 +125,6 @@ let build_artifacts ] } -let publish_to_debian_repo = - \(spec : MinaBuildSpec.Type) - -> Command.build - Command.Config::{ - , commands = - Toolchain.select - spec.toolchainSelectMode - spec.debVersion - [ "AWS_ACCESS_KEY_ID" - , "AWS_SECRET_ACCESS_KEY" - , "MINA_DEB_CODENAME=${DebianVersions.lowerName - spec.debVersion}" - , "MINA_DEB_RELEASE=${DebianChannel.lowerName spec.channel}" - ] - "./buildkite/scripts/debian/publish.sh" - , label = - "Publish Mina for ${DebianVersions.capitalName - spec.debVersion} ${Profiles.toSuffixUppercase - spec.profile}" - , key = "publish-deb-pkg" - , depends_on = - DebianVersions.dependsOnStep - (Some spec.prefix) - spec.debVersion - spec.profile - spec.buildFlags - "build" - , target = Size.Small - } - let docker_step : Artifacts.Type -> MinaBuildSpec.Type -> List DockerImage.ReleaseSpec.Type = \(artifact : Artifacts.Type) @@ -145,116 +135,97 @@ let docker_step DebianVersions.dependsOnStep (Some spec.prefix) spec.debVersion + spec.network spec.profile spec.buildFlags step_dep_name in merge { Daemon = - Prelude.List.map - Network.Type - DockerImage.ReleaseSpec.Type - ( \(n : Network.Type) - -> DockerImage.ReleaseSpec::{ - , deps = deps - , service = - Artifacts.dockerName Artifacts.Type.Daemon - , network = Network.lowerName n - , deb_codename = - "${DebianVersions.lowerName spec.debVersion}" - , deb_profile = spec.profile - , build_flags = spec.buildFlags - , deb_repo = DebianRepo.Type.Local - , step_key = - "daemon-${Network.lowerName - n}-${DebianVersions.lowerName - spec.debVersion}${Profiles.toLabelSegment - spec.profile}${BuildFlags.toLabelSegment - spec.buildFlags}-docker-image" - } - ) - spec.networks + [ DockerImage.ReleaseSpec::{ + , deps = deps + , service = Artifacts.Type.Daemon + , network = Network.lowerName spec.network + , deb_codename = spec.debVersion + , deb_profile = spec.profile + , build_flags = spec.buildFlags + , deb_repo = DebianRepo.Type.Local + } + ] , TestExecutive = [] : List DockerImage.ReleaseSpec.Type , LogProc = [] : List DockerImage.ReleaseSpec.Type + , Toolchain = + [ DockerImage.ReleaseSpec::{ + , service = Artifacts.Type.Toolchain + , network = Network.lowerName Network.Type.Devnet + , deb_codename = spec.debVersion + , deb_profile = spec.profile + , build_flags = spec.buildFlags + , deb_repo = DebianRepo.Type.Local + } + ] + , ItnOrchestrator = + [ DockerImage.ReleaseSpec::{ + , service = Artifacts.Type.ItnOrchestrator + , network = Network.lowerName Network.Type.Devnet + , deb_repo = DebianRepo.Type.Local + } + ] + , Leaderboard = + [ DockerImage.ReleaseSpec::{ + , service = Artifacts.Type.Leaderboard + } + ] , BatchTxn = [ DockerImage.ReleaseSpec::{ , deps = deps - , service = "mina-batch-txn" - , network = "berkeley" - , deb_codename = - "${DebianVersions.lowerName spec.debVersion}" + , service = Artifacts.Type.BatchTxn + , network = Network.lowerName spec.network + , deb_codename = spec.debVersion , deb_profile = spec.profile , build_flags = spec.buildFlags , deb_repo = DebianRepo.Type.Local - , step_key = - "batch-txn-${DebianVersions.lowerName - spec.debVersion}${BuildFlags.toLabelSegment - spec.buildFlags}--docker-image" } ] , Archive = [ DockerImage.ReleaseSpec::{ , deps = deps - , service = "mina-archive" - , deb_codename = - "${DebianVersions.lowerName spec.debVersion}" + , service = Artifacts.Type.Archive + , deb_codename = spec.debVersion , deb_profile = spec.profile , build_flags = spec.buildFlags , deb_repo = DebianRepo.Type.Local - , step_key = - "archive-${DebianVersions.lowerName - spec.debVersion}${Profiles.toLabelSegment - spec.profile}${BuildFlags.toLabelSegment - spec.buildFlags}-docker-image" } ] , Rosetta = [ DockerImage.ReleaseSpec::{ , deps = deps - , service = "mina-rosetta" - , network = "berkeley" - , build_flags = spec.buildFlags - , deb_repo = DebianRepo.Type.Local + , service = Artifacts.Type.Rosetta + , network = Network.lowerName spec.network + , deb_codename = spec.debVersion , deb_profile = spec.profile - , deb_codename = - "${DebianVersions.lowerName spec.debVersion}" - , step_key = - "rosetta-${DebianVersions.lowerName - spec.debVersion}${BuildFlags.toLabelSegment - spec.buildFlags}-docker-image" + , deb_repo = DebianRepo.Type.Local } ] , ZkappTestTransaction = [ DockerImage.ReleaseSpec::{ , deps = deps - , service = "mina-zkapp-test-transaction" + , service = Artifacts.Type.ZkappTestTransaction , build_flags = spec.buildFlags , deb_repo = DebianRepo.Type.Local , deb_profile = spec.profile - , deb_codename = - "${DebianVersions.lowerName spec.debVersion}" - , step_key = - "zkapp-test-transaction-${DebianVersions.lowerName - spec.debVersion}${Profiles.toLabelSegment - spec.profile}${BuildFlags.toLabelSegment - spec.buildFlags}--docker-image" + , deb_codename = spec.debVersion } ] , FunctionalTestSuite = [ DockerImage.ReleaseSpec::{ , deps = deps - , service = "mina-test-suite" - , deb_codename = - "${DebianVersions.lowerName spec.debVersion}" + , service = Artifacts.Type.FunctionalTestSuite + , network = Network.lowerName Network.Type.Devnet + , deb_codename = spec.debVersion , build_flags = spec.buildFlags , deb_repo = DebianRepo.Type.Local , deb_profile = spec.profile - , step_key = - "test-suite-${DebianVersions.lowerName - spec.debVersion}${Profiles.toLabelSegment - spec.profile}${BuildFlags.toLabelSegment - spec.buildFlags}--docker-image" - , network = "berkeley" } ] } @@ -289,6 +260,34 @@ let docker_commands ) flattened_docker_steps +let publish_to_debian_repo = + \(spec : MinaBuildSpec.Type) + -> Command.build + Command.Config::{ + , commands = + Toolchain.select + spec.toolchainSelectMode + spec.debVersion + [ "AWS_ACCESS_KEY_ID" + , "AWS_SECRET_ACCESS_KEY" + , "MINA_DEB_CODENAME=${DebianVersions.lowerName + spec.debVersion}" + , "MINA_DEB_RELEASE=${DebianChannel.lowerName spec.channel}" + ] + "./buildkite/scripts/debian/publish.sh" + , label = "Debian: Publish ${labelSuffix spec}" + , key = "publish-deb-pkg" + , depends_on = + DebianVersions.dependsOnStep + (Some spec.prefix) + spec.debVersion + spec.network + spec.profile + spec.buildFlags + "build" + , target = Size.Small + } + let pipelineBuilder : MinaBuildSpec.Type -> List Command.Type -> Pipeline.Config.Type = \(spec : MinaBuildSpec.Type) @@ -297,11 +296,7 @@ let pipelineBuilder , spec = JobSpec::{ , dirtyWhen = DebianVersions.dirtyWhen spec.debVersion , path = "Release" - , name = - "${spec.prefix}${DebianVersions.capitalName - spec.debVersion}${Profiles.toSuffixUppercase - spec.profile}${BuildFlags.toSuffixUppercase - spec.buildFlags}" + , name = "${spec.prefix}${nameSuffix spec}" , tags = spec.tags , mode = spec.mode } @@ -333,4 +328,6 @@ let pipeline in { pipeline = pipeline , onlyDebianPipeline = onlyDebianPipeline , MinaBuildSpec = MinaBuildSpec + , labelSuffix = labelSuffix + , keySuffix = keySuffix } diff --git a/buildkite/src/Command/PromotePackage.dhall b/buildkite/src/Command/PromotePackage.dhall new file mode 100644 index 00000000000..4989004d394 --- /dev/null +++ b/buildkite/src/Command/PromotePackage.dhall @@ -0,0 +1,305 @@ +let B = ../External/Buildkite.dhall + +let B/If = B.definitions/commandStep/properties/if/Type + +let Prelude = ../External/Prelude.dhall + +let List/map = Prelude.List.map + +let Package = ../Constants/DebianPackage.dhall + +let Network = ../Constants/Network.dhall + +let PipelineMode = ../Pipeline/Mode.dhall + +let PipelineTag = ../Pipeline/Tag.dhall + +let Pipeline = ../Pipeline/Dsl.dhall + +let JobSpec = ../Pipeline/JobSpec.dhall + +let DebianChannel = ../Constants/DebianChannel.dhall + +let Profiles = ../Constants/Profiles.dhall + +let Artifact = ../Constants/Artifacts.dhall + +let DebianVersions = ../Constants/DebianVersions.dhall + +let Toolchain = ../Constants/Toolchain.dhall + +let Command = ./Base.dhall + +let Size = ./Size.dhall + +let Cmd = ../Lib/Cmds.dhall + +let PromoteDebianSpec = + { Type = + { deps : List Command.TaggedKey.Type + , package : Package.Type + , version : Text + , new_version : Text + , architecture : Text + , network : Network.Type + , codename : DebianVersions.DebVersion + , from_channel : DebianChannel.Type + , to_channel : DebianChannel.Type + , profile : Profiles.Type + , remove_profile_from_name : Bool + , step_key : Text + , if : Optional B/If + } + , default = + { deps = [] : List Command.TaggedKey.Type + , package = Package.Type.LogProc + , version = "" + , new_version = "" + , architecture = "amd64" + , network = Network.Type.Devnet + , codename = DebianVersions.DebVersion.Bullseye + , from_channel = DebianChannel.Type.Unstable + , to_channel = DebianChannel.Type.NightlyDevelop + , profile = Profiles.Type.Standard + , remove_profile_from_name = False + , step_key = "promote-debian-package" + , if = None B/If + } + } + +let PromoteDockerSpec = + { Type = + { deps : List Command.TaggedKey.Type + , name : Artifact.Type + , version : Text + , profile : Profiles.Type + , codename : DebianVersions.DebVersion + , new_tag : Text + , network : Network.Type + , step_key : Text + , if : Optional B/If + , publish : Bool + , remove_profile_from_name : Bool + } + , default = + { deps = [] : List Command.TaggedKey.Type + , name = Artifact.Type.Daemon + , version = "" + , new_tag = "" + , step_key = "promote-docker" + , profile = Profiles.Type.Standard + , network = Network.Type.Devnet + , codename = DebianVersions.DebVersion.Bullseye + , if = None B/If + , publish = False + , remove_profile_from_name = False + } + } + +let promoteDebianStep = + \(spec : PromoteDebianSpec.Type) + -> let package_name + : Text + = Package.debianName spec.package spec.profile spec.network + + let new_name = + if spec.remove_profile_from_name + + then "--new-name ${Package.debianName + spec.package + Profiles.Type.Standard + spec.network}" + + else "" + + in Command.build + Command.Config::{ + , commands = + Toolchain.runner + DebianVersions.DebVersion.Bullseye + [ "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY" ] + "./buildkite/scripts/promote-deb.sh --package ${package_name} --version ${spec.version} --new-version ${spec.new_version} --architecture ${spec.architecture} --codename ${DebianVersions.lowerName + spec.codename} --from-component ${DebianChannel.lowerName + spec.from_channel} --to-component ${DebianChannel.lowerName + spec.to_channel} ${new_name}" + , label = "Debian: ${spec.step_key}" + , key = spec.step_key + , target = Size.XLarge + , depends_on = spec.deps + , if = spec.if + } + +let promoteDebianVerificationStep = + \(spec : PromoteDebianSpec.Type) + -> let name = + if spec.remove_profile_from_name + + then "${Package.debianName + spec.package + Profiles.Type.Standard + spec.network}" + + else Package.debianName spec.package spec.profile spec.network + + in Command.build + Command.Config::{ + , commands = + [ Cmd.run + "./scripts/debian/verify.sh --package ${name} --version ${spec.new_version} --codename ${DebianVersions.lowerName + spec.codename} --channel ${DebianChannel.lowerName + spec.to_channel}" + ] + , label = "Debian: ${spec.step_key}" + , key = spec.step_key + , target = Size.Small + , depends_on = spec.deps + , if = spec.if + } + +let promoteDockerStep = + \(spec : PromoteDockerSpec.Type) + -> let old_tag = + Artifact.dockerTag + spec.name + spec.version + spec.codename + spec.profile + spec.network + False + + let new_tag = + Artifact.dockerTag + spec.name + spec.new_tag + spec.codename + spec.profile + spec.network + spec.remove_profile_from_name + + let publish = if spec.publish then "-p" else "" + + in Command.build + Command.Config::{ + , commands = + [ Cmd.run + "./buildkite/scripts/promote-docker.sh --name ${Artifact.dockerName + spec.name} --version ${old_tag} --tag ${new_tag} ${publish}" + ] + , label = "Docker: ${spec.step_key}" + , key = spec.step_key + , target = Size.XLarge + , depends_on = spec.deps + , if = spec.if + } + +let promoteDockerVerificationStep = + \(spec : PromoteDockerSpec.Type) + -> let new_tag = + Artifact.dockerTag + spec.name + spec.new_tag + spec.codename + spec.profile + spec.network + spec.remove_profile_from_name + + let repo = + if spec.publish + + then "docker.io/minaprotocol" + + else "gcr.io/o1labs-192920" + + in Command.build + Command.Config::{ + , commands = + [ Cmd.run + "docker pull ${repo}/${Artifact.dockerName + spec.name}:${new_tag}" + ] + , label = "Docker: ${spec.step_key}" + , key = spec.step_key + , target = Size.Small + , depends_on = spec.deps + , if = spec.if + } + +let promotePipeline + : List PromoteDebianSpec.Type + -> List PromoteDockerSpec.Type + -> DebianVersions.DebVersion + -> PipelineMode.Type + -> Pipeline.Config.Type + = \(debians_spec : List PromoteDebianSpec.Type) + -> \(dockers_spec : List PromoteDockerSpec.Type) + -> \(debVersion : DebianVersions.DebVersion) + -> \(mode : PipelineMode.Type) + -> let steps = + List/map + PromoteDebianSpec.Type + Command.Type + (\(spec : PromoteDebianSpec.Type) -> promoteDebianStep spec) + debians_spec + # List/map + PromoteDockerSpec.Type + Command.Type + (\(spec : PromoteDockerSpec.Type) -> promoteDockerStep spec) + dockers_spec + + in Pipeline.Config::{ + , spec = JobSpec::{ + , dirtyWhen = DebianVersions.dirtyWhen debVersion + , path = "Release" + , name = "PromotePackage" + , tags = [] : List PipelineTag.Type + , mode = mode + } + , steps = steps + } + +let verifyPipeline + : List PromoteDebianSpec.Type + -> List PromoteDockerSpec.Type + -> DebianVersions.DebVersion + -> PipelineMode.Type + -> Pipeline.Config.Type + = \(debians_spec : List PromoteDebianSpec.Type) + -> \(dockers_spec : List PromoteDockerSpec.Type) + -> \(debVersion : DebianVersions.DebVersion) + -> \(mode : PipelineMode.Type) + -> let steps = + List/map + PromoteDebianSpec.Type + Command.Type + ( \(spec : PromoteDebianSpec.Type) + -> promoteDebianVerificationStep spec + ) + debians_spec + # List/map + PromoteDockerSpec.Type + Command.Type + ( \(spec : PromoteDockerSpec.Type) + -> promoteDockerVerificationStep spec + ) + dockers_spec + + in Pipeline.Config::{ + , spec = JobSpec::{ + , dirtyWhen = DebianVersions.dirtyWhen debVersion + , path = "Release" + , name = "VerifyPackage" + , tags = [] : List PipelineTag.Type + , mode = mode + } + , steps = steps + } + +in { promoteDebianStep = promoteDebianStep + , promoteDockerStep = promoteDockerStep + , promoteDebianVerificationStep = promoteDebianVerificationStep + , promoteDockerVerificationStep = promoteDockerVerificationStep + , promotePipeline = promotePipeline + , verifyPipeline = verifyPipeline + , PromoteDebianSpec = PromoteDebianSpec + , PromoteDockerSpec = PromoteDockerSpec + } diff --git a/buildkite/src/Command/Promotion/PromoteDebian.dhall b/buildkite/src/Command/Promotion/PromoteDebian.dhall index 0019779bbba..2ac01eb006a 100644 --- a/buildkite/src/Command/Promotion/PromoteDebian.dhall +++ b/buildkite/src/Command/Promotion/PromoteDebian.dhall @@ -40,7 +40,7 @@ let PromoteDebianSpec = , version = "\\\\\$MINA_DEB_VERSION" , new_version = "\\\\\$MINA_DEB_VERSION" , architecture = "amd64" - , network = Network.Type.Berkeley + , network = Network.Type.Devnet , codename = DebianVersions.DebVersion.Bullseye , from_channel = DebianChannel.Type.Unstable , to_channel = DebianChannel.Type.NightlyCompatible diff --git a/buildkite/src/Command/Promotion/PromoteDocker.dhall b/buildkite/src/Command/Promotion/PromoteDocker.dhall index cbc60627dc8..7e56225d5ef 100644 --- a/buildkite/src/Command/Promotion/PromoteDocker.dhall +++ b/buildkite/src/Command/Promotion/PromoteDocker.dhall @@ -41,7 +41,7 @@ let PromoteDockerSpec = , new_tags = [] : List Text , step_key = "promote-docker" , profile = Profiles.Type.Standard - , network = Network.Type.Berkeley + , network = Network.Type.Devnet , codename = DebianVersions.DebVersion.Bullseye , if = None B/If , publish = False diff --git a/buildkite/src/Command/ReplayerTest.dhall b/buildkite/src/Command/ReplayerTest.dhall index 6a742c7e7fd..d93f70a2677 100644 --- a/buildkite/src/Command/ReplayerTest.dhall +++ b/buildkite/src/Command/ReplayerTest.dhall @@ -17,7 +17,7 @@ in { step = ([] : List Text) "./src/test/archive/sample_db/archive_db.sql" Artifacts.Type.Archive - (None Network.Type) + Network.Type.Devnet "./buildkite/scripts/replayer-test.sh" ] , label = "Archive: Replayer test" diff --git a/buildkite/src/Command/RunWithPostgres.dhall b/buildkite/src/Command/RunWithPostgres.dhall index b6279112a23..a9ad935999e 100644 --- a/buildkite/src/Command/RunWithPostgres.dhall +++ b/buildkite/src/Command/RunWithPostgres.dhall @@ -4,10 +4,6 @@ let P = Prelude let Text/concatMap = P.Text.concatMap -let Optional/map = Prelude.Optional.map - -let Optional/default = Prelude.Optional.default - let Cmd = ../Lib/Cmds.dhall let ContainerImages = ../Constants/ContainerImages.dhall @@ -17,16 +13,11 @@ let Artifacts = ../Constants/Artifacts.dhall let Network = ../Constants/Network.dhall let runInDockerWithPostgresConn - : List Text - -> Text - -> Artifacts.Type - -> Optional Network.Type - -> Text - -> Cmd.Type + : List Text -> Text -> Artifacts.Type -> Network.Type -> Text -> Cmd.Type = \(environment : List Text) -> \(initScript : Text) -> \(docker : Artifacts.Type) - -> \(network : Optional Network.Type) + -> \(network : Network.Type) -> \(innerScript : Text) -> let port = "5432" @@ -64,15 +55,6 @@ let runInDockerWithPostgresConn : Text = "\\\$MINA_DOCKER_TAG" - let maybeNetwork = - Optional/map - Network.Type - Text - (\(network : Network.Type) -> "-${Network.lowerName network}") - network - - let networkOrDefault = Optional/default Text "" maybeNetwork - in Cmd.chain [ "( docker stop ${postgresDockerName} && docker rm ${postgresDockerName} ) || true" , "source buildkite/scripts/export-git-env-vars.sh" @@ -80,7 +62,7 @@ let runInDockerWithPostgresConn , "sleep 5" , "docker exec ${postgresDockerName} psql ${pg_conn} -f /workdir/${initScript}" , "docker run --network host --volume ${outerDir}:/workdir --workdir /workdir --entrypoint bash ${envVars} gcr.io/o1labs-192920/${Artifacts.dockerName - docker}:${minaDockerTag}${networkOrDefault} ${innerScript}" + docker}:${minaDockerTag} ${innerScript}" ] in { runInDockerWithPostgresConn = runInDockerWithPostgresConn } diff --git a/buildkite/src/Constants/Artifacts.dhall b/buildkite/src/Constants/Artifacts.dhall index b41f52cf28c..673b14c3808 100644 --- a/buildkite/src/Constants/Artifacts.dhall +++ b/buildkite/src/Constants/Artifacts.dhall @@ -18,6 +18,9 @@ let Artifact | Rosetta | ZkappTestTransaction | FunctionalTestSuite + | Toolchain + | ItnOrchestrator + | Leaderboard > let AllButTests = @@ -33,7 +36,13 @@ let AllButTests = let Main = [ Artifact.Daemon, Artifact.LogProc, Artifact.Archive, Artifact.Rosetta ] -let All = AllButTests # [ Artifact.FunctionalTestSuite ] +let All = + AllButTests + # [ Artifact.FunctionalTestSuite + , Artifact.Toolchain + , Artifact.ItnOrchestrator + , Artifact.Leaderboard + ] let capitalName = \(artifact : Artifact) @@ -46,6 +55,9 @@ let capitalName = , Rosetta = "Rosetta" , ZkappTestTransaction = "ZkappTestTransaction" , FunctionalTestSuite = "FunctionalTestSuite" + , Toolchain = "Toolchain" + , ItnOrchestrator = "ItnOrchestrator" + , Leaderboard = "Leaderboard" } artifact @@ -60,6 +72,9 @@ let lowerName = , Rosetta = "rosetta" , ZkappTestTransaction = "zkapp_test_transaction" , FunctionalTestSuite = "functional_test_suite" + , Toolchain = "toolchain" + , ItnOrchestrator = "itnOrchestrator" + , Leaderboard = "leaderboard" } artifact @@ -74,6 +89,9 @@ let dockerName = , Rosetta = "mina-rosetta" , ZkappTestTransaction = "mina-zkapp-test-transaction" , FunctionalTestSuite = "mina-test-suite" + , Toolchain = "mina-toolchain" + , ItnOrchestrator = "itn-orchestrator" + , Leaderboard = "leaderboard" } artifact @@ -89,38 +107,32 @@ let toDebianName = , Rosetta = "rosetta_${Network.lowerName network}" , ZkappTestTransaction = "zkapp_test_transaction" , FunctionalTestSuite = "functional_test_suite" + , Toolchain = "" + , ItnOrchestrator = "" + , Leaderboard = "" } artifact let toDebianNames = \(artifacts : List Artifact) - -> \(networks : List Network.Type) + -> \(network : Network.Type) -> let list_of_list_of_debians = Prelude.List.map Artifact (List Text) ( \(a : Artifact) -> merge - { Daemon = - Prelude.List.map - Network.Type - Text - (\(n : Network.Type) -> toDebianName a n) - networks + { Daemon = [ toDebianName a network ] , Archive = [ "archive" ] , LogProc = [ "logproc" ] , TestExecutive = [ "test_executive" ] , BatchTxn = [ "batch_txn" ] - , Rosetta = - Prelude.List.map - Network.Type - Text - ( \(n : Network.Type) - -> "rosetta_${Network.lowerName n}" - ) - networks + , Rosetta = [ toDebianName a network ] , ZkappTestTransaction = [ "zkapp_test_transaction" ] , FunctionalTestSuite = [ "functional_test_suite" ] + , Toolchain = [] : List Text + , ItnOrchestrator = [] : List Text + , Leaderboard = [] : List Text } a ) @@ -165,6 +177,9 @@ let dockerTag = "${version_and_codename}-${Network.lowerName network}" , ZkappTestTransaction = "${version_and_codename}" , FunctionalTestSuite = "${version_and_codename}" + , Toolchain = "${version_and_codename}" + , ItnOrchestrator = "${version_and_codename}" + , Leaderboard = "${version_and_codename}" } artifact diff --git a/buildkite/src/Constants/DebianVersions.dhall b/buildkite/src/Constants/DebianVersions.dhall index f3042f5f067..ba4bb08311b 100644 --- a/buildkite/src/Constants/DebianVersions.dhall +++ b/buildkite/src/Constants/DebianVersions.dhall @@ -4,6 +4,8 @@ let Optional/default = Prelude.Optional.default let Profiles = ./Profiles.dhall +let Network = ./Network.dhall + let BuildFlags = ./BuildFlags.dhall let S = ../Lib/SelectFiles.dhall @@ -33,6 +35,7 @@ let lowerName = let dependsOnStep = \(prefix : Optional Text) -> \(debVersion : DebVersion) + -> \(network : Network.Type) -> \(profile : Profiles.Type) -> \(buildFlag : BuildFlags.Type) -> \(step : Text) @@ -40,47 +43,28 @@ let dependsOnStep = let prefix = Optional/default Text "MinaArtifact" prefix + let name = + "${prefix}${capitalName + debVersion}${Network.capitalName + network}${profileSuffix}${BuildFlags.toSuffixUppercase + buildFlag}" + in merge - { Bookworm = - [ { name = - "${prefix}${profileSuffix}${BuildFlags.toSuffixUppercase - buildFlag}" - , key = "${step}-deb-pkg" - } - ] - , Bullseye = - [ { name = - "${prefix}${capitalName - debVersion}${profileSuffix}${BuildFlags.toSuffixUppercase - buildFlag}" - , key = "${step}-deb-pkg" - } - ] - , Jammy = - [ { name = - "${prefix}${capitalName - debVersion}${profileSuffix}${BuildFlags.toSuffixUppercase - buildFlag}" - , key = "${step}-deb-pkg" - } - ] - , Focal = - [ { name = - "${prefix}${capitalName - debVersion}${profileSuffix}${BuildFlags.toSuffixUppercase - buildFlag}" - , key = "${step}-deb-pkg" - } - ] + { Bookworm = [ { name = name, key = "${step}-deb-pkg" } ] + , Bullseye = [ { name = name, key = "${step}-deb-pkg" } ] + , Jammy = [ { name = name, key = "${step}-deb-pkg" } ] + , Focal = [ { name = name, key = "${step}-deb-pkg" } ] } debVersion let dependsOn = \(debVersion : DebVersion) + -> \(network : Network.Type) -> \(profile : Profiles.Type) -> dependsOnStep (None Text) debVersion + network profile BuildFlags.Type.None "build" diff --git a/buildkite/src/Constants/DockerVersions.dhall b/buildkite/src/Constants/DockerVersions.dhall index 7db6179132c..926a077b77d 100644 --- a/buildkite/src/Constants/DockerVersions.dhall +++ b/buildkite/src/Constants/DockerVersions.dhall @@ -1,9 +1,3 @@ -let Prelude = ../External/Prelude.dhall - -let Optional/map = Prelude.Optional.map - -let Optional/default = Prelude.Optional.default - let Profiles = ./Profiles.dhall let Artifacts = ./Artifacts.dhall @@ -36,46 +30,47 @@ let lowerName = let dependsOn = \(docker : Docker) - -> \(network : Optional Network.Type) + -> \(network : Network.Type) -> \(profile : Profiles.Type) -> \(binary : Artifacts.Type) - -> let profileSuffix = Profiles.toSuffixUppercase profile + -> let network = "${Network.capitalName network}" + + let profileSuffix = "${Profiles.toSuffixUppercase profile}" let prefix = "MinaArtifact" let suffix = "docker-image" - let maybeNetwork = - Optional/map - Network.Type - Text - (\(network : Network.Type) -> "-${Network.lowerName network}") - network - - let networkOrDefault = Optional/default Text "" maybeNetwork - - let key = - "${Artifacts.lowerName - binary}${networkOrDefault}-${lowerName docker}-${suffix}" + let key = "${Artifacts.lowerName binary}-${suffix}" in merge { Bookworm = - [ { name = "${prefix}${capitalName docker}${profileSuffix}" + [ { name = + "${prefix}${capitalName + docker}${network}${profileSuffix}" , key = key } ] , Bullseye = - [ { name = "${prefix}${capitalName docker}${profileSuffix}" + [ { name = + "${prefix}${capitalName + docker}${network}${profileSuffix}" , key = key } ] , Jammy = - [ { name = "${prefix}${capitalName docker}${profileSuffix}" + [ { name = + "${prefix}${capitalName + docker}${network}${capitalName + docker}${profileSuffix}" , key = key } ] , Focal = - [ { name = "${prefix}${capitalName docker}${profileSuffix}" + [ { name = + "${prefix}${capitalName + docker}${network}${capitalName + docker}${profileSuffix}" , key = key } ] diff --git a/buildkite/src/Constants/Network.dhall b/buildkite/src/Constants/Network.dhall index 92462c87ae3..86094577017 100644 --- a/buildkite/src/Constants/Network.dhall +++ b/buildkite/src/Constants/Network.dhall @@ -4,23 +4,18 @@ let List/any = Prelude.List.any let Network : Type - = < Devnet | Mainnet | Berkeley > + = < Devnet | Mainnet > let capitalName = \(network : Network) - -> merge - { Devnet = "Devnet", Mainnet = "Mainnet", Berkeley = "Berkeley" } - network + -> merge { Devnet = "Devnet", Mainnet = "Mainnet" } network let lowerName = \(network : Network) - -> merge - { Devnet = "devnet", Mainnet = "mainnet", Berkeley = "berkeley" } - network + -> merge { Devnet = "devnet", Mainnet = "mainnet" } network let requiresMainnetBuild = - \(network : Network) - -> merge { Devnet = True, Mainnet = True, Berkeley = False } network + \(network : Network) -> merge { Devnet = False, Mainnet = True } network let foldMinaBuildMainnetEnv = \(networks : List Network) @@ -30,9 +25,28 @@ let foldMinaBuildMainnetEnv = else "MINA_BUILD_MAINNET=false" +let buildMainnetEnv = + \(network : Network) + -> if requiresMainnetBuild network + + then "MINA_BUILD_MAINNET=true" + + else "MINA_BUILD_MAINNET=false" + +let foldNames = + \(networks : List Network) + -> Prelude.List.fold + Network + networks + Text + (\(x : Network) -> \(y : Text) -> "${capitalName x}" ++ y) + "" + in { Type = Network , capitalName = capitalName , lowerName = lowerName , requiresMainnetBuild = requiresMainnetBuild , foldMinaBuildMainnetEnv = foldMinaBuildMainnetEnv + , buildMainnetEnv = buildMainnetEnv + , foldNames = foldNames } diff --git a/buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall b/buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall deleted file mode 100644 index 5c88037f8d1..00000000000 --- a/buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall +++ /dev/null @@ -1,46 +0,0 @@ -let S = ../../Lib/SelectFiles.dhall - -let Pipeline = ../../Pipeline/Dsl.dhall - -let PipelineTag = ../../Pipeline/Tag.dhall - -let JobSpec = ../../Pipeline/JobSpec.dhall - -let DockerImage = ../../Command/DockerImage.dhall - -let Profiles = ../../Constants/Profiles.dhall - -let DebianVersions = ../../Constants/DebianVersions.dhall - -let DebianRepo = ../../Constants/DebianRepo.dhall - -let spec = - DockerImage.ReleaseSpec::{ - , service = "itn-orchestrator" - , step_key = "itn-orchestrator-docker-image" - , network = "berkeley" - , deb_repo = DebianRepo.Type.Local - , deps = - DebianVersions.dependsOn - DebianVersions.DebVersion.Bullseye - Profiles.Type.Standard - } - -in Pipeline.build - Pipeline.Config::{ - , spec = JobSpec::{ - , dirtyWhen = - [ S.strictlyStart - (S.contains "buildkite/src/Jobs/Release/ItnOrchestratorArtifact") - , S.strictlyStart (S.contains "src/app/itn_orchestrator") - ] - , path = "Release" - , name = "ItnOrchestratorArtifact" - , tags = - [ PipelineTag.Type.Long - , PipelineTag.Type.Release - , PipelineTag.Type.Stable - ] - } - , steps = [ DockerImage.generateStep spec ] - } diff --git a/buildkite/src/Jobs/Release/LeaderboardArtifact.dhall b/buildkite/src/Jobs/Release/LeaderboardArtifact.dhall deleted file mode 100644 index 2482ee5b0d3..00000000000 --- a/buildkite/src/Jobs/Release/LeaderboardArtifact.dhall +++ /dev/null @@ -1,54 +0,0 @@ -let Cmd = ../../Lib/Cmds.dhall - -let S = ../../Lib/SelectFiles.dhall - -let Pipeline = ../../Pipeline/Dsl.dhall - -let PipelineTag = ../../Pipeline/Tag.dhall - -let JobSpec = ../../Pipeline/JobSpec.dhall - -let Command = ../../Command/Base.dhall - -let Size = ../../Command/Size.dhall - -let DockerImage = ../../Command/DockerImage.dhall - -let spec = - DockerImage.ReleaseSpec::{ - , service = "leaderboard" - , step_key = "leaderboard-docker-image" - } - -in Pipeline.build - Pipeline.Config::{ - , spec = JobSpec::{ - , dirtyWhen = - [ S.strictlyStart - (S.contains "buildkite/src/Jobs/Release/LeaderboardArtifact") - , S.strictlyStart (S.contains "frontend/leaderboard") - ] - , path = "Release" - , name = "LeaderboardArtifact" - , tags = - [ PipelineTag.Type.Long - , PipelineTag.Type.Release - , PipelineTag.Type.Stable - ] - } - , steps = - [ Command.build - Command.Config::{ - , commands = - [ Cmd.run - "echo export MINA_VERSION=\$(cat frontend/leaderboard/package.json | jq '.version') > LEADERBOARD_DEPLOY_ENV" - ] - , label = "Setup Leaderboard docker image deploy environment" - , key = "setup-deploy-env" - , target = Size.Small - , artifact_paths = - [ S.contains "frontend/leaderboard/package.json" ] - } - , DockerImage.generateStep spec - ] - } diff --git a/buildkite/src/Jobs/Release/MinaArtifactBullseye.dhall b/buildkite/src/Jobs/Release/MinaArtifactBullseyeDevnet.dhall similarity index 100% rename from buildkite/src/Jobs/Release/MinaArtifactBullseye.dhall rename to buildkite/src/Jobs/Release/MinaArtifactBullseyeDevnet.dhall diff --git a/buildkite/src/Jobs/Release/MinaArtifactBullseyeInstrumented.dhall b/buildkite/src/Jobs/Release/MinaArtifactBullseyeDevnetInstrumented.dhall similarity index 100% rename from buildkite/src/Jobs/Release/MinaArtifactBullseyeInstrumented.dhall rename to buildkite/src/Jobs/Release/MinaArtifactBullseyeDevnetInstrumented.dhall diff --git a/buildkite/src/Jobs/Release/MinaArtifactBullseyeLightnet.dhall b/buildkite/src/Jobs/Release/MinaArtifactBullseyeDevnetLightnet.dhall similarity index 100% rename from buildkite/src/Jobs/Release/MinaArtifactBullseyeLightnet.dhall rename to buildkite/src/Jobs/Release/MinaArtifactBullseyeDevnetLightnet.dhall diff --git a/buildkite/src/Jobs/Release/MinaArtifactMainnetBullseye.dhall b/buildkite/src/Jobs/Release/MinaArtifactBullseyeMainnet.dhall similarity index 88% rename from buildkite/src/Jobs/Release/MinaArtifactMainnetBullseye.dhall rename to buildkite/src/Jobs/Release/MinaArtifactBullseyeMainnet.dhall index aa4037d8626..18633217d3c 100644 --- a/buildkite/src/Jobs/Release/MinaArtifactMainnetBullseye.dhall +++ b/buildkite/src/Jobs/Release/MinaArtifactBullseyeMainnet.dhall @@ -21,13 +21,12 @@ in Pipeline.build , Artifacts.Type.Rosetta , Artifacts.Type.ZkappTestTransaction ] - , networks = [ Network.Type.Devnet, Network.Type.Mainnet ] + , network = Network.Type.Mainnet , tags = [ PipelineTag.Type.Long , PipelineTag.Type.Release , PipelineTag.Type.Stable ] , mode = PipelineMode.Type.Stable - , prefix = "MinaArtifactMainnet" } ) diff --git a/buildkite/src/Jobs/Release/MinaArtifactFocal.dhall b/buildkite/src/Jobs/Release/MinaArtifactFocalDevnet.dhall similarity index 100% rename from buildkite/src/Jobs/Release/MinaArtifactFocal.dhall rename to buildkite/src/Jobs/Release/MinaArtifactFocalDevnet.dhall diff --git a/buildkite/src/Jobs/Release/MinaArtifactMainnetFocal.dhall b/buildkite/src/Jobs/Release/MinaArtifactFocalMainnet.dhall similarity index 89% rename from buildkite/src/Jobs/Release/MinaArtifactMainnetFocal.dhall rename to buildkite/src/Jobs/Release/MinaArtifactFocalMainnet.dhall index 82d9d49a532..76829c42836 100644 --- a/buildkite/src/Jobs/Release/MinaArtifactMainnetFocal.dhall +++ b/buildkite/src/Jobs/Release/MinaArtifactFocalMainnet.dhall @@ -24,13 +24,12 @@ in Pipeline.build , Artifacts.Type.ZkappTestTransaction ] , debVersion = DebianVersions.DebVersion.Focal - , networks = [ Network.Type.Devnet, Network.Type.Mainnet ] + , network = Network.Type.Mainnet , tags = [ PipelineTag.Type.Long , PipelineTag.Type.Release , PipelineTag.Type.Stable ] , mode = PipelineMode.Type.Stable - , prefix = "MinaArtifactMainnet" } ) diff --git a/buildkite/src/Jobs/Release/MinaArtifactOnlyDebianBullseye.dhall b/buildkite/src/Jobs/Release/MinaArtifactOnlyDebianBullseyeDevnet.dhall similarity index 100% rename from buildkite/src/Jobs/Release/MinaArtifactOnlyDebianBullseye.dhall rename to buildkite/src/Jobs/Release/MinaArtifactOnlyDebianBullseyeDevnet.dhall diff --git a/buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall b/buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall index 86e483e1e31..357da774d05 100644 --- a/buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall +++ b/buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall @@ -8,6 +8,10 @@ let JobSpec = ../../Pipeline/JobSpec.dhall let DockerImage = ../../Command/DockerImage.dhall +let DebianVersions = ../../Constants/DebianVersions.dhall + +let Artifacts = ../../Constants/Artifacts.dhall + in Pipeline.build Pipeline.Config::{ , spec = JobSpec::{ @@ -27,10 +31,9 @@ in Pipeline.build , steps = [ let toolchainBullseyeSpec = DockerImage.ReleaseSpec::{ - , service = "mina-toolchain" - , deb_codename = "bullseye" + , service = Artifacts.Type.Toolchain + , deb_codename = DebianVersions.DebVersion.Bullseye , no_cache = True - , step_key = "toolchain-bullseye-docker-image" } in DockerImage.generateStep toolchainBullseyeSpec diff --git a/buildkite/src/Jobs/Release/MinaToolchainArtifactFocal.dhall b/buildkite/src/Jobs/Release/MinaToolchainArtifactFocal.dhall index ab0030f99f4..2ffe0472892 100644 --- a/buildkite/src/Jobs/Release/MinaToolchainArtifactFocal.dhall +++ b/buildkite/src/Jobs/Release/MinaToolchainArtifactFocal.dhall @@ -8,6 +8,10 @@ let JobSpec = ../../Pipeline/JobSpec.dhall let DockerImage = ../../Command/DockerImage.dhall +let DebianVersions = ../../Constants/DebianVersions.dhall + +let Artifacts = ../../Constants/Artifacts.dhall + in Pipeline.build Pipeline.Config::{ , spec = JobSpec::{ @@ -25,14 +29,13 @@ in Pipeline.build , tags = [ PipelineTag.Type.Toolchain ] } , steps = - [ let toolchainSpec = + [ let toolchainBullseyeSpec = DockerImage.ReleaseSpec::{ - , service = "mina-toolchain" - , deb_codename = "focal" + , service = Artifacts.Type.Toolchain + , deb_codename = DebianVersions.DebVersion.Focal , no_cache = True - , step_key = "toolchain-focal-docker-image" } - in DockerImage.generateStep toolchainSpec + in DockerImage.generateStep toolchainBullseyeSpec ] } diff --git a/buildkite/src/Jobs/Test/BerkeleyCompatibility.dhall b/buildkite/src/Jobs/Test/BerkeleyCompatibility.dhall deleted file mode 100644 index b44d2b6ea96..00000000000 --- a/buildkite/src/Jobs/Test/BerkeleyCompatibility.dhall +++ /dev/null @@ -1,48 +0,0 @@ -let JobSpec = ../../Pipeline/JobSpec.dhall - -let Pipeline = ../../Pipeline/Dsl.dhall - -let PipelineTag = ../../Pipeline/Tag.dhall - -let Cmd = ../../Lib/Cmds.dhall - -let S = ../../Lib/SelectFiles.dhall - -let Command = ../../Command/Base.dhall - -let Docker = ../../Command/Docker/Type.dhall - -let Size = ../../Command/Size.dhall - -let dependsOn = - [ { name = "MinaArtifactBullseye" - , key = "daemon-berkeley-bullseye-docker-image" - } - ] - -in Pipeline.build - Pipeline.Config::{ - , spec = JobSpec::{ - , dirtyWhen = - [ S.strictlyStart (S.contains "src") - , S.exactly "buildkite/scripts/check-compatibility" "sh" - , S.exactly "buildkite/src/Jobs/Test/BerkeleyCompatibility" "dhall" - ] - , path = "Test" - , tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] - , name = "BerkeleyCompatibility" - } - , steps = - [ Command.build - Command.Config::{ - , commands = - [ Cmd.run "buildkite/scripts/check-compatibility.sh berkeley" ] - , label = "Test: berkeley compatibilty test" - , key = "berkeley-compatibilty-test" - , target = Size.XLarge - , docker = None Docker.Type - , depends_on = dependsOn - , timeout_in_minutes = Some +60 - } - ] - } diff --git a/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall b/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall index dee29949cd5..bef4ebb1e47 100644 --- a/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall +++ b/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall @@ -12,9 +12,12 @@ let DebianVersions = ../../Constants/DebianVersions.dhall let Profiles = ../../Constants/Profiles.dhall +let Network = ../../Constants/Network.dhall + let dependsOn = DebianVersions.dependsOn DebianVersions.DebVersion.Bullseye + Network.Type.Devnet Profiles.Type.Standard in Pipeline.build diff --git a/buildkite/src/Jobs/Test/ConnectToBerkeley.dhall b/buildkite/src/Jobs/Test/ConnectToDevnet.dhall similarity index 74% rename from buildkite/src/Jobs/Test/ConnectToBerkeley.dhall rename to buildkite/src/Jobs/Test/ConnectToDevnet.dhall index 312e63f9435..526ee022143 100644 --- a/buildkite/src/Jobs/Test/ConnectToBerkeley.dhall +++ b/buildkite/src/Jobs/Test/ConnectToDevnet.dhall @@ -14,18 +14,15 @@ let ConnectToTestnet = ../../Command/ConnectToTestnet.dhall let Profiles = ../../Constants/Profiles.dhall -let Artifacts = ../../Constants/Artifacts.dhall +let DebianVersions = ../../Constants/DebianVersions.dhall let Network = ../../Constants/Network.dhall -let Dockers = ../../Constants/DockerVersions.dhall - let dependsOn = - Dockers.dependsOn - Dockers.Type.Bullseye - (Some Network.Type.Berkeley) + DebianVersions.dependsOn + DebianVersions.DebVersion.Bullseye + Network.Type.Devnet Profiles.Type.Standard - Artifacts.Type.Daemon in Pipeline.build Pipeline.Config::{ @@ -33,17 +30,17 @@ in Pipeline.build , dirtyWhen = [ S.strictlyStart (S.contains "src") , S.exactly "buildkite/scripts/connect-to-testnet" "sh" - , S.exactly "buildkite/src/Jobs/Test/ConnectToBerkeley" "dhall" + , S.exactly "buildkite/src/Jobs/Test/ConnectToDevnet" "dhall" , S.exactly "buildkite/src/Command/ConnectToTestnet" "dhall" ] , path = "Test" - , name = "ConnectToBerkeley" + , name = "ConnectToDevnet" , tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] } , steps = [ ConnectToTestnet.step dependsOn - "berkeley" + "${Network.lowerName Network.Type.Devnet}" "40s" "2m" (B/SoftFail.Boolean True) diff --git a/buildkite/src/Jobs/Test/DevelopCompatibility.dhall b/buildkite/src/Jobs/Test/DevelopCompatibility.dhall index 99132224a5b..8bcecf6c27f 100644 --- a/buildkite/src/Jobs/Test/DevelopCompatibility.dhall +++ b/buildkite/src/Jobs/Test/DevelopCompatibility.dhall @@ -14,11 +14,17 @@ let Docker = ../../Command/Docker/Type.dhall let Size = ../../Command/Size.dhall +let DebianVersions = ../../Constants/DebianVersions.dhall + +let Network = ../../Constants/Network.dhall + +let Profiles = ../../Constants/Profiles.dhall + let dependsOn = - [ { name = "MinaArtifactBullseye" - , key = "daemon-berkeley-bullseye-docker-image" - } - ] + DebianVersions.dependsOn + DebianVersions.DebVersion.Bullseye + Network.Type.Devnet + Profiles.Type.Lightnet in Pipeline.build Pipeline.Config::{ diff --git a/buildkite/src/Jobs/Test/ReplayerTest.dhall b/buildkite/src/Jobs/Test/ReplayerTest.dhall index 0e3d665e2ce..24ce7acf7e1 100644 --- a/buildkite/src/Jobs/Test/ReplayerTest.dhall +++ b/buildkite/src/Jobs/Test/ReplayerTest.dhall @@ -19,7 +19,7 @@ let Artifacts = ../../Constants/Artifacts.dhall let dependsOn = Dockers.dependsOn Dockers.Type.Bullseye - (None Network.Type) + Network.Type.Devnet Profiles.Type.Standard Artifacts.Type.Archive diff --git a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall index 4d7ae4b9c3b..02ed47087de 100644 --- a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall +++ b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall @@ -18,14 +18,16 @@ let Profiles = ../../Constants/Profiles.dhall let Dockers = ../../Constants/DockerVersions.dhall -let Artifacts = ../../Constants/Artifacts.dhall +let RunWithPostgres = ../../Command/RunWithPostgres.dhall let Network = ../../Constants/Network.dhall -let RunWithPostgres = ../../Command/RunWithPostgres.dhall +let Artifacts = ../../Constants/Artifacts.dhall let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type +let network = Network.Type.Devnet + let dirtyWhen = [ S.strictlyStart (S.contains "src") , S.exactly "buildkite/src/Jobs/Test/RosettaIntegrationTests" "dhall" @@ -55,12 +57,13 @@ in Pipeline.build ([] : List Text) "./src/test/archive/sample_db/archive_db.sql" Artifacts.Type.Rosetta - (Some Network.Type.Berkeley) + Network.Type.Devnet "./buildkite/scripts/rosetta-indexer-test.sh" , Cmd.runInDocker Cmd.Docker::{ , image = - "gcr.io/o1labs-192920/mina-rosetta:\\\${MINA_DOCKER_TAG}-berkeley" + "gcr.io/o1labs-192920/mina-rosetta:\\\${MINA_DOCKER_TAG}-${Network.lowerName + network}" } "buildkite/scripts/rosetta-integration-tests-fast.sh" ] @@ -71,7 +74,7 @@ in Pipeline.build , depends_on = Dockers.dependsOn Dockers.Type.Bullseye - (None Network.Type) + network Profiles.Type.Standard Artifacts.Type.Rosetta } diff --git a/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall b/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall index 10ff09fea4e..ebfea0150c1 100644 --- a/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall +++ b/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall @@ -16,16 +16,18 @@ let Command = ../../Command/Base.dhall let Size = ../../Command/Size.dhall -let Network = ../../Constants/Network.dhall - let Profiles = ../../Constants/Profiles.dhall -let Artifacts = ../../Constants/Artifacts.dhall - let Dockers = ../../Constants/DockerVersions.dhall +let Network = ../../Constants/Network.dhall + +let Artifacts = ../../Constants/Artifacts.dhall + let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type +let network = Network.Type.Devnet + let dirtyWhen = [ S.strictlyStart (S.contains "src") , S.exactly "buildkite/src/Jobs/Test/RosettaIntegrationTests" "dhall" @@ -55,7 +57,8 @@ in Pipeline.build , Cmd.runInDocker Cmd.Docker::{ , image = - "gcr.io/o1labs-192920/mina-rosetta:\\\${MINA_DOCKER_TAG}-berkeley" + "gcr.io/o1labs-192920/mina-rosetta:\\\${MINA_DOCKER_TAG}-${Network.lowerName + network}" } "buildkite/scripts/rosetta-integration-tests-full.sh" ] @@ -66,7 +69,7 @@ in Pipeline.build , depends_on = Dockers.dependsOn Dockers.Type.Bullseye - (None Network.Type) + network Profiles.Type.Standard Artifacts.Type.Rosetta } diff --git a/buildkite/src/Jobs/Test/RunSnarkProfiler.dhall b/buildkite/src/Jobs/Test/RunSnarkProfiler.dhall index 0523f1e1e74..accae521318 100644 --- a/buildkite/src/Jobs/Test/RunSnarkProfiler.dhall +++ b/buildkite/src/Jobs/Test/RunSnarkProfiler.dhall @@ -16,11 +16,14 @@ let RunInToolchain = ../../Command/RunInToolchain.dhall let Docker = ../../Command/Docker/Type.dhall +let Network = ../../Constants/Network.dhall + let Size = ../../Command/Size.dhall let dependsOn = DebianVersions.dependsOn DebianVersions.DebVersion.Bullseye + Network.Type.Devnet Profiles.Type.Standard let buildTestCmd diff --git a/buildkite/src/Jobs/Test/SingleNodeTest.dhall b/buildkite/src/Jobs/Test/SingleNodeTest.dhall index 74959dd325a..e5bbae08182 100644 --- a/buildkite/src/Jobs/Test/SingleNodeTest.dhall +++ b/buildkite/src/Jobs/Test/SingleNodeTest.dhall @@ -14,6 +14,8 @@ let DebianVersions = ../../Constants/DebianVersions.dhall let Profiles = ../../Constants/Profiles.dhall +let Network = ../../Constants/Network.dhall + let Docker = ../../Command/Docker/Type.dhall let Size = ../../Command/Size.dhall @@ -21,9 +23,11 @@ let Size = ../../Command/Size.dhall let dependsOn = DebianVersions.dependsOn DebianVersions.DebVersion.Bullseye + Network.Type.Devnet Profiles.Type.Lightnet # DebianVersions.dependsOn DebianVersions.DebVersion.Bullseye + Network.Type.Devnet Profiles.Type.Standard let buildTestCmd diff --git a/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall b/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall index 933a21bae6e..a388d2e0220 100644 --- a/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall +++ b/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall @@ -21,12 +21,12 @@ let Artifacts = ../../Constants/Artifacts.dhall let dependsOn = Dockers.dependsOn Dockers.Type.Bullseye - (Some Network.Type.Berkeley) + Network.Type.Devnet Profiles.Type.Standard Artifacts.Type.Daemon # Dockers.dependsOn Dockers.Type.Bullseye - (None Network.Type) + Network.Type.Devnet Profiles.Type.Standard Artifacts.Type.Archive diff --git a/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall b/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall index c02eb9c5262..85a8b614ee2 100644 --- a/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall +++ b/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall @@ -21,12 +21,12 @@ let Artifacts = ../../Constants/Artifacts.dhall let dependsOn = Dockers.dependsOn Dockers.Type.Bullseye - (Some Network.Type.Berkeley) + Network.Type.Devnet Profiles.Type.Standard Artifacts.Type.Daemon # Dockers.dependsOn Dockers.Type.Bullseye - (None Network.Type) + Network.Type.Devnet Profiles.Type.Standard Artifacts.Type.Archive diff --git a/buildkite/src/Jobs/Test/VersionLint.dhall b/buildkite/src/Jobs/Test/VersionLint.dhall index bc47db104ad..971dd12d44e 100644 --- a/buildkite/src/Jobs/Test/VersionLint.dhall +++ b/buildkite/src/Jobs/Test/VersionLint.dhall @@ -20,7 +20,17 @@ let Docker = ../../Command/Docker/Type.dhall let Size = ../../Command/Size.dhall -let dependsOn = [ { name = "MinaArtifactBullseye", key = "build-deb-pkg" } ] +let DebianVersions = ../../Constants/DebianVersions.dhall + +let Network = ../../Constants/Network.dhall + +let Profiles = ../../Constants/Profiles.dhall + +let dependsOn = + DebianVersions.dependsOn + DebianVersions.DebVersion.Bullseye + Network.Type.Devnet + Profiles.Type.Standard let buildTestCmd : Text -> Size -> List Command.TaggedKey.Type -> B/SoftFail -> Command.Type diff --git a/scripts/debian/builder-helpers.sh b/scripts/debian/builder-helpers.sh index 0e34178731d..0cfd2d219c7 100755 --- a/scripts/debian/builder-helpers.sh +++ b/scripts/debian/builder-helpers.sh @@ -10,7 +10,7 @@ MINA_DEB_RELEASE=${MINA_DEB_RELEASE:-"unstable"} # Helper script to include when building deb archives. -echo "--- Setting up the envrionment to build debian packages..." +echo "--- Setting up the environment to build debian packages..." SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" cd "${SCRIPTPATH}/../../_build" @@ -46,13 +46,13 @@ esac # Add suffix to debian to distinguish different profiles (mainnet/devnet/lightnet) case "${DUNE_PROFILE}" in devnet|mainnet) - MINA_DEB_NAME="mina-berkeley" + MINA_DEB_NAME="mina-devnet" DEB_SUFFIX="" ;; *) # use dune profile as suffix but replace underscore to dashes so deb builder won't complain _SUFFIX=${DUNE_PROFILE//_/-} - MINA_DEB_NAME="mina-berkeley-${_SUFFIX}" + MINA_DEB_NAME="mina-devnet-${_SUFFIX}" DEB_SUFFIX="-${_SUFFIX}" ;; esac @@ -325,28 +325,13 @@ build_daemon_devnet_deb() { echo "------------------------------------------------------------" echo "--- Building testnet signatures deb without keys:" - create_control_file mina-devnet "${SHARED_DEPS}${DAEMON_DEPS}" 'Mina Protocol Client and Daemon for the Devnet Network' "${SUGGESTED_DEPS}" + create_control_file "${MINA_DEB_NAME}" "${SHARED_DEPS}${DAEMON_DEPS}" 'Mina Protocol Client and Daemon for the Devnet Network' copy_common_daemon_configs devnet testnet 'seed-lists/devnet_seeds.txt' - build_deb mina-devnet -} -##################################### END DEVNET PACKAGE ####################################### - -##################################### BERKELEY PACKAGE ####################################### -build_daemon_berkeley_deb() { - - echo "------------------------------------------------------------" - echo "--- Building Mina Berkeley testnet signatures deb without keys:" - - create_control_file "${MINA_DEB_NAME}" "${SHARED_DEPS}${DAEMON_DEPS}" 'Mina Protocol Client and Daemon' - - copy_common_daemon_configs berkeley testnet 'seed-lists/berkeley_seeds.txt' - build_deb "${MINA_DEB_NAME}" - } -##################################### END BERKELEY PACKAGE ####################################### +##################################### END DEVNET PACKAGE ####################################### ##################################### ARCHIVE PACKAGE ########################################## build_archive_deb () { diff --git a/scripts/docker/helper.sh b/scripts/docker/helper.sh index 07cf02af46a..312b79657ec 100644 --- a/scripts/docker/helper.sh +++ b/scripts/docker/helper.sh @@ -43,6 +43,18 @@ function export_suffixes () { ;; esac ;; + lightnet) + case "${DEB_BUILD_FLAGS}" in + *instrumented) + export DOCKER_DEB_SUFFIX="--build-arg deb_suffix=lightnet-instrumented" + export BUILD_FLAG_SUFFIX="lightnet-instrumented" + ;; + *) + export DOCKER_DEB_SUFFIX="--build-arg deb_suffix=lightnet" + export BUILD_FLAG_SUFFIX="-lightnet" + ;; + esac + ;; *) case "${DEB_BUILD_FLAGS}" in *instrumented) diff --git a/src/lib/cache_dir/native/cache_dir.ml b/src/lib/cache_dir/native/cache_dir.ml index 8450d40bfd7..d76cd00aa38 100644 --- a/src/lib/cache_dir/native/cache_dir.ml +++ b/src/lib/cache_dir/native/cache_dir.ml @@ -55,7 +55,7 @@ let load_from_s3 s3_bucket_prefix s3_install_path ~logger = @@ Monitor.try_with ~here:[%here] (fun () -> let each_uri (uri_string, file_path) = let open Deferred.Let_syntax in - [%log trace] "Downloading file from S3" + [%log trace] "Downloading file from S3: $url to $local_file_path" ~metadata: [ ("url", `String uri_string) ; ("local_file_path", `String file_path) diff --git a/src/lib/integration_test_lib/test_config.ml b/src/lib/integration_test_lib/test_config.ml index fa06f656fc3..18297518678 100644 --- a/src/lib/integration_test_lib/test_config.ml +++ b/src/lib/integration_test_lib/test_config.ml @@ -88,14 +88,14 @@ type t = let proof_config_default : Runtime_config.Proof_keys.t = { level = Some Full - ; sub_windows_per_window = None - ; ledger_depth = None - ; work_delay = None + ; sub_windows_per_window = Some 11 + ; ledger_depth = Some 35 + ; work_delay = Some 2 ; block_window_duration_ms = Some 120000 - ; transaction_capacity = None - ; coinbase_amount = None - ; supercharged_coinbase_factor = None - ; account_creation_fee = None + ; transaction_capacity = Some (Log_2 7) + ; coinbase_amount = Some (Currency.Amount.of_mina_int_exn 720) + ; supercharged_coinbase_factor = Some 1 + ; account_creation_fee = Some (Currency.Fee.of_mina_string_exn "1") ; fork = None } diff --git a/src/lib/integration_test_local_engine/docker_node_config.ml b/src/lib/integration_test_local_engine/docker_node_config.ml index 16bba170998..f207055960d 100644 --- a/src/lib/integration_test_local_engine/docker_node_config.ml +++ b/src/lib/integration_test_local_engine/docker_node_config.ml @@ -105,6 +105,7 @@ module Base_node_config = struct mina accounts import -config-directory /root/.mina-config -privkey-path "$key_file" fi done + rm /var/lib/coda/config* # Execute the puppeteer script exec /mina_daemon_puppeteer.py "$@" |} diff --git a/src/lib/integration_test_local_engine/mina_docker.ml b/src/lib/integration_test_local_engine/mina_docker.ml index 9a92fe18304..f0f15c20b7e 100644 --- a/src/lib/integration_test_local_engine/mina_docker.ml +++ b/src/lib/integration_test_local_engine/mina_docker.ml @@ -158,16 +158,16 @@ module Network_config = struct Some { txpool_max_size = Some txpool_max_size ; peer_list_url = None - ; zkapp_proof_update_cost = None - ; zkapp_signed_single_update_cost = None - ; zkapp_signed_pair_update_cost = None - ; zkapp_transaction_cost_limit = None - ; max_event_elements = None - ; max_action_elements = None - ; zkapp_cmd_limit_hardcap = None + ; zkapp_proof_update_cost = Some 10.26 + ; zkapp_signed_single_update_cost = Some 9.140000000000001 + ; zkapp_signed_pair_update_cost = Some 10.08 + ; zkapp_transaction_cost_limit = Some 69.45 + ; max_event_elements = Some 100 + ; max_action_elements = Some 100 + ; zkapp_cmd_limit_hardcap = Some 128 ; slot_tx_end ; slot_chain_end - ; minimum_user_command_fee = None + ; minimum_user_command_fee = Some (Currency.Fee.of_string "1000000") ; network_id } ; genesis =