Skip to content

Commit

Permalink
Merge branch 'develop' into georgeee/remove-caqti-toolchain-images-de…
Browse files Browse the repository at this point in the history
…velop
  • Loading branch information
dkijania committed Jun 20, 2024
2 parents 8ea3fd5 + 6ec8abb commit 024d623
Show file tree
Hide file tree
Showing 16 changed files with 1,272 additions and 808 deletions.
6 changes: 3 additions & 3 deletions buildkite/scripts/export-git-env-vars.sh
Original file line number Diff line number Diff line change
Expand Up @@ -55,10 +55,10 @@ export MINA_DOCKER_TAG="$(echo "${MINA_DEB_VERSION}-${MINA_DEB_CODENAME}" | sed

# Determine the packages to build (mainnet y/N)
case $GITBRANCH in
compatible|master|release/1*) # whitelist of branches that are "mainnet-like"
MINA_BUILD_MAINNET=true ;;
compatible|master|release-automation-testing/*|release/1*|release/3*) # whitelist of branches that are "mainnet-like"
export MINA_BUILD_MAINNET=true ;;
*) # Other branches
MINA_BUILD_MAINNET=false ;;
export MINA_BUILD_MAINNET=false ;;
esac

echo "Publishing on release channel \"${RELEASE}\""
Expand Down
2 changes: 1 addition & 1 deletion buildkite/scripts/promote-deb.sh
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ echo "Promoting debs: ${PACKAGE}_${VERSION} to Release: ${TO_COMPONENT} and Code
# If this fails, attempt to remove the lockfile and retry.

if [[ -z "$NEW_VERSION" ]] || [[ "$NEW_VERSION" == "$VERSION" ]]; then
deb-s3 copy --s3-region=us-west-2 --bucket packages.o1test.net --preserve-versions --cache-control=max-age=120 $PACKAGE $CODENAME $TO_COMPONENT --versions $VERSION --arch $ARCH --component ${FROM_COMPONENT} --codename ${CODENAME}
deb-s3 copy --s3-region=us-west-2 --lock --bucket packages.o1test.net --preserve-versions --cache-control=max-age=120 $PACKAGE $CODENAME $TO_COMPONENT --versions $VERSION --arch $ARCH --component ${FROM_COMPONENT} --codename ${CODENAME}
else
source scripts/reversion-deb.sh \
--deb $PACKAGE \
Expand Down
110 changes: 110 additions & 0 deletions buildkite/scripts/run_verify_promoted_build_job.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
#!/bin/bash

# Usage (in buildkite definition)

# steps:
# - commands:
# - "./buildkite/scripts/run_promote_build_job.sh | buildkite-agent pipeline upload"
# label: ":pipeline: run promote dockers build job"
# agents:
# size: "generic"
# plugins:
# "docker#v3.5.0":
# environment:
# - BUILDKITE_AGENT_ACCESS_TOKEN
# - "DOCKERS=Archive,Daemon"
# - "REMOVE_PROFILE_FROM_NAME=1"
# - "PROFILE=Hardfork"
# - "NETWORK=Devnet"
# - "FROM_VERSION=3.0.0devnet-tooling-dkijania-hardfork-package-gen-in-nightly-b37f50e"
# - "NEW_VERSION=3.0.0fake-ddb6fc4"
# - "CODENAMES=Focal,Buster,Bullseye"
# - "FROM_CHANNEL=Unstable"
# - "TO_CHANNEL=Experimental"
# image: codaprotocol/ci-toolchain-base:v3
# mount-buildkite-agent: true
# propagate-environment: true


DEBIAN_DHALL_DEF="(./buildkite/src/Constants/DebianPackage.dhall)"
DOCKER_DHALL_DEF="(./buildkite/src/Constants/Artifacts.dhall)"
DEBIAN_VERSION_DHALL_DEF="(./buildkite/src/Constants/DebianVersions.dhall)"
PROMOTE_PACKAGE_DHALL_DEF="(./buildkite/src/Entrypoints/PromotePackage.dhall)"
PROFILES_DHALL_DEF="(./buildkite/src/Constants/Profiles.dhall)"
NETWORK_DHALL_DEF="(./buildkite/src/Constants/Network.dhall)"
DEBIAN_CHANNEL_DHALL_DEF="(./buildkite/src/Constants/DebianChannel.dhall)"


function usage() {
if [[ -n "$1" ]]; then
echo -e "${RED}$1${CLEAR}\n";
fi
echo " DEBIANS The comma delimitered debian names. For example: 'Daemon,Archive' "
echo " DOCKERS The comma delimitered docker names. For example: 'Daemon,Archive' "
echo " CODENAMES The Debian codenames (Bullseye, Buster etc.)"
echo " NEW_VERSION The new Debian version or new Docker tag"
echo " REMOVE_PROFILE_FROM_NAME Should we remove profile suffix from debian name"
echo " PROFILE The Docker and Debian profile (Standard, Lightnet)"
echo " NETWORK The Docker and Debian network (Devnet, Mainnet, Berkeley)"
echo " TO_CHANNEL Target debian channel"
echo " PUBLISH The Publish to docker.io flag. If defined, script will publish docker do docker.io. Otherwise it will still resides in gcr.io"
echo ""
exit 1
}

if [ -z "$DEBIANS" ] && [ -z "$DOCKERS" ]; then usage "No Debians nor Dockers defined for promoting!"; exit 1; fi;

DHALL_DEBIANS="([] : List $DEBIAN_DHALL_DEF.Type)"

if [[ -n "$DEBIANS" ]]; then
if [[ -z "$CODENAMES" ]]; then usage "Codenames is not set!"; exit 1; fi;
if [[ -z "$PROFILE" ]]; then PROFILE="Standard"; fi;
if [[ -z "$NETWORK" ]]; then NETWORK="Berkeley"; fi;
if [[ -z "$REMOVE_PROFILE_FROM_NAME" ]]; then REMOVE_PROFILE_FROM_NAME=0; fi;
if [[ -z "$PUBLISH" ]]; then PUBLISH=0; fi;
if [[ -z "$TO_CHANNEL" ]]; then TO_CHANNEL="Unstable"; fi;
if [[ -z "$NEW_VERSION" ]]; then NEW_VERSION=$FROM_VERSION; fi;


arr_of_debians=(${DEBIANS//,/ })
DHALL_DEBIANS=""
for i in "${arr_of_debians[@]}"; do
DHALL_DEBIANS="${DHALL_DEBIANS}, $DEBIAN_DHALL_DEF.Type.${i}"
done
DHALL_DEBIANS="[${DHALL_DEBIANS:1}]"
fi


DHALL_DOCKERS="([] : List $DOCKER_DHALL_DEF.Type)"

if [[ $PUBLISH -eq 1 ]]; then
DHALL_PUBLISH="True"
else
DHALL_PUBLISH="False"
fi

if [[ -n "$DOCKERS" ]]; then
if [[ -z "$NEW_VERSION" ]]; then usage "New Tag is not set!"; fi;
if [[ -z "$PROFILE" ]]; then PROFILE="Standard"; fi;

arr_of_dockers=(${DOCKERS//,/ })
DHALL_DOCKERS=""
for i in "${arr_of_dockers[@]}"; do
DHALL_DOCKERS="${DHALL_DOCKERS}, $DOCKER_DHALL_DEF.Type.${i}"
done
DHALL_DOCKERS="[${DHALL_DOCKERS:1}]"
fi

CODENAMES=(${CODENAMES//,/ })
DHALL_CODENAMES=""
for i in "${CODENAMES[@]}"; do
DHALL_CODENAMES="${DHALL_CODENAMES}, $DEBIAN_VERSION_DHALL_DEF.DebVersion.${i}"
done
DHALL_CODENAMES="[${DHALL_CODENAMES:1}]"

if [[ "${REMOVE_PROFILE_FROM_NAME}" -eq 0 ]]; then
REMOVE_PROFILE_FROM_NAME="False"
else
REMOVE_PROFILE_FROM_NAME="True"
fi
echo $PROMOTE_PACKAGE_DHALL_DEF'.verify_artifacts '"$DHALL_DEBIANS"' '"$DHALL_DOCKERS"' "'"${NEW_VERSION}"'" '$PROFILES_DHALL_DEF'.Type.'"${PROFILE}"' '$NETWORK_DHALL_DEF'.Type.'"${NETWORK}"' '"${DHALL_CODENAMES}"' '$DEBIAN_CHANNEL_DHALL_DEF'.Type.'"${TO_CHANNEL}"' "'"${TAG}"'" '${REMOVE_PROFILE_FROM_NAME}' '${DHALL_PUBLISH}' ' | dhall-to-yaml --quoted
77 changes: 75 additions & 2 deletions buildkite/src/Command/PromotePackage.dhall
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,21 @@ let promoteDebianStep = \(spec : PromoteDebianSpec.Type) ->
`if` = spec.`if`
}

let promoteDebianVerificationStep = \(spec : PromoteDebianSpec.Type) ->
let name = if spec.remove_profile_from_name then "${Package.debianName spec.package Profiles.Type.Standard spec.network}" else (Package.debianName spec.package spec.profile spec.network)
in
Command.build
Command.Config::{
commands = [
Cmd.run "./scripts/debian/verify.sh --package ${name} --version ${spec.new_version} --codename ${DebianVersions.lowerName spec.codename} --channel ${DebianChannel.lowerName spec.to_channel}"
],
label = "Debian: ${spec.step_key}",
key = spec.step_key,
target = Size.Small,
depends_on = spec.deps,
`if` = spec.`if`
}

let promoteDockerStep = \(spec : PromoteDockerSpec.Type) ->
let old_tag = Artifact.dockerTag spec.name spec.version spec.codename spec.profile spec.network False
let new_tag = Artifact.dockerTag spec.name spec.new_tag spec.codename spec.profile spec.network spec.remove_profile_from_name
Expand All @@ -121,8 +136,24 @@ let promoteDockerStep = \(spec : PromoteDockerSpec.Type) ->
`if` = spec.`if`
}

let promoteDockerVerificationStep = \(spec : PromoteDockerSpec.Type) ->
let new_tag = Artifact.dockerTag spec.name spec.new_tag spec.codename spec.profile spec.network spec.remove_profile_from_name
let repo = if spec.publish then "docker.io/minaprotocol" else "gcr.io/o1labs-192920"
in
Command.build
Command.Config::{
commands = [
Cmd.run "docker pull ${repo}/${Artifact.dockerName spec.name}:${new_tag}"
],
label = "Docker: ${spec.step_key}",
key = spec.step_key,
target = Size.Small,
depends_on = spec.deps,
`if` = spec.`if`
}


let pipeline : List PromoteDebianSpec.Type ->
let promotePipeline : List PromoteDebianSpec.Type ->
List PromoteDockerSpec.Type ->
DebianVersions.DebVersion ->
PipelineMode.Type ->
Expand Down Expand Up @@ -159,11 +190,53 @@ let pipeline : List PromoteDebianSpec.Type ->
},
steps = steps
}

let verifyPipeline : List PromoteDebianSpec.Type ->
List PromoteDockerSpec.Type ->
DebianVersions.DebVersion ->
PipelineMode.Type ->
Pipeline.Config.Type =
\(debians_spec : List PromoteDebianSpec.Type) ->
\(dockers_spec : List PromoteDockerSpec.Type) ->
\(debVersion: DebianVersions.DebVersion) ->
\(mode: PipelineMode.Type) ->

let steps =
(List/map
PromoteDebianSpec.Type
Command.Type
(\(spec: PromoteDebianSpec.Type ) -> promoteDebianVerificationStep spec )
debians_spec
)
#
(List/map
PromoteDockerSpec.Type
Command.Type
(\(spec: PromoteDockerSpec.Type ) -> promoteDockerVerificationStep spec )
dockers_spec
)
in
--- tags are empty on purpose not to allow run job from automatically triggered pipelines
Pipeline.Config::{
spec =
JobSpec::{
dirtyWhen = DebianVersions.dirtyWhen debVersion,
path = "Release",
name = "VerifyPackage",
tags = []: List PipelineTag.Type,
mode = mode
},
steps = steps
}

in

{ promoteDebianStep = promoteDebianStep
, promoteDockerStep = promoteDockerStep
, pipeline = pipeline
, promoteDebianVerificationStep = promoteDebianVerificationStep
, promoteDockerVerificationStep = promoteDockerVerificationStep
, promotePipeline = promotePipeline
, verifyPipeline = verifyPipeline
, PromoteDebianSpec = PromoteDebianSpec
, PromoteDockerSpec = PromoteDockerSpec
}
99 changes: 97 additions & 2 deletions buildkite/src/Entrypoints/PromotePackage.dhall
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,101 @@ let promote_artifacts =

let pipelineType = Pipeline.build
(
PromotePackage.pipeline
PromotePackage.promotePipeline
(debians_spec)
(dockers_spec)
(DebianVersions.DebVersion.Bullseye)
(PipelineMode.Type.Stable)
)
in pipelineType.pipeline

let verify_artifacts =
\(debians: List Package.Type) ->
\(dockers: List Artifact.Type) ->
\(new_version: Text ) ->
\(profile: Profile.Type) ->
\(network: Network.Type) ->
\(codenames: List DebianVersions.DebVersion ) ->
\(to_channel: DebianChannel.Type ) ->
\(tag: Text ) ->
\(remove_profile_from_name: Bool) ->
\(publish: Bool) ->

let debians_spec =
List/map
Package.Type
(List PromotePackage.PromoteDebianSpec.Type)
(\(debian: Package.Type) ->
List/map
DebianVersions.DebVersion
PromotePackage.PromoteDebianSpec.Type
(\(codename: DebianVersions.DebVersion) ->
PromotePackage.PromoteDebianSpec::{
profile = profile
, package = debian
, new_version = new_version
, network = network
, codename = codename
, to_channel = to_channel
, remove_profile_from_name = remove_profile_from_name
, step_key = "verify-promote-debian-${Package.lowerName debian}-${DebianVersions.lowerName codename}-${DebianChannel.lowerName to_channel}"
}
)
codenames

)
debians
in

let debians_spec =
Prelude.List.fold
(List PromotePackage.PromoteDebianSpec.Type)
debians_spec
(List PromotePackage.PromoteDebianSpec.Type)
(\(a : List PromotePackage.PromoteDebianSpec.Type) -> \(b : List PromotePackage.PromoteDebianSpec.Type) -> a # b)
([] : List PromotePackage.PromoteDebianSpec.Type)

in

let dockers_spec =
List/map
Artifact.Type
(List PromotePackage.PromoteDockerSpec.Type)
(\(docker: Artifact.Type) ->
List/map
DebianVersions.DebVersion
PromotePackage.PromoteDockerSpec.Type
(\(codename: DebianVersions.DebVersion) ->
PromotePackage.PromoteDockerSpec::{
profile = profile
, name = docker
, codename = codename
, new_tag = new_version
, network = network
, publish = publish
, remove_profile_from_name = remove_profile_from_name
, step_key = "verify-tag-${Artifact.lowerName docker}-${DebianVersions.lowerName codename}-docker"
}
)
codenames
)
dockers

in

let dockers_spec =
Prelude.List.fold
(List PromotePackage.PromoteDockerSpec.Type)
dockers_spec
(List PromotePackage.PromoteDockerSpec.Type)
(\(a : List PromotePackage.PromoteDockerSpec.Type) -> \(b : List PromotePackage.PromoteDockerSpec.Type) -> a # b)
([] : List PromotePackage.PromoteDockerSpec.Type)

in

let pipelineType = Pipeline.build
(
PromotePackage.verifyPipeline
(debians_spec)
(dockers_spec)
(DebianVersions.DebVersion.Bullseye)
Expand All @@ -113,5 +207,6 @@ let promote_artifacts =
in pipelineType.pipeline

in {
promote_artifacts = promote_artifacts
promote_artifacts = promote_artifacts,
verify_artifacts = verify_artifacts
}
7 changes: 3 additions & 4 deletions buildkite/src/Jobs/Lint/Merge.dhall
Original file line number Diff line number Diff line change
Expand Up @@ -48,10 +48,9 @@ Pipeline.build
},
Command.build
Command.Config::{
commands = [ Cmd.run "buildkite/scripts/merges-cleanly.sh berkeley"]
, label = "Check merges cleanly into berkeley"
, key = "clean-merge-berkeley"
, soft_fail = Some (B/SoftFail.Boolean True)
commands = [ Cmd.run "buildkite/scripts/merges-cleanly.sh master"]
, label = "Check merges cleanly into master"
, key = "clean-merge-master"
, target = Size.Small
, docker = Some Docker::{
image = (../../Constants/ContainerImages.dhall).toolchainBase
Expand Down
Loading

0 comments on commit 024d623

Please sign in to comment.