Merge pull request #13782 from MinaProtocol/dkijania/mina-profile-docker

[CI] extract profile settings and build docker based on its values

buildkite/src/Command/DockerImage.dhall

@@ -20,6 +20,7 @@ let ReleaseSpec = {
     deb_codename: Text,
     deb_release: Text,
     deb_version: Text,
+    deb_profile: Text,
     extra_args: Text,
     step_key: Text
   },
@@ -30,10 +31,11 @@ let ReleaseSpec = {
     service = "\\\${MINA_SERVICE}",
     branch = "\\\${BUILDKITE_BRANCH}",
     deb_codename = "bullseye",
+    deb_profile = "devnet",
     deb_release = "\\\${MINA_DEB_RELEASE}",
     deb_version = "\\\${MINA_DEB_VERSION}",
     extra_args = "",
-    step_key = "daemon-devnet-docker-image"
+    step_key = "daemon-standard-docker-image"
   }
 }
 
@@ -43,7 +45,7 @@ let generateStep = \(spec : ReleaseSpec.Type) ->
     [
         Cmd.run (
           "export MINA_DEB_CODENAME=${spec.deb_codename} && source ./buildkite/scripts/export-git-env-vars.sh && ./scripts/release-docker.sh " ++
-              "--service ${spec.service} --version ${spec.version} --network ${spec.network} --branch ${spec.branch} --deb-codename ${spec.deb_codename} --deb-release ${spec.deb_release} --deb-version ${spec.deb_version} --extra-args \\\"${spec.extra_args}\\\""
+              "--service ${spec.service} --version ${spec.version} --network ${spec.network} --branch ${spec.branch} --deb-codename ${spec.deb_codename} --deb-release ${spec.deb_release} --deb-version ${spec.deb_version} --deb-profile ${spec.deb_profile} --extra-args \\\"${spec.extra_args}\\\""
         )
     ]
 

buildkite/src/Command/MinaArtifact.dhall

@@ -16,34 +16,36 @@ let Size = ./Size.dhall
 let Libp2p = ./Libp2pHelperBuild.dhall
 let DockerImage = ./DockerImage.dhall
 let DebianVersions = ../Constants/DebianVersions.dhall
+let Profiles = ../Constants/Profiles.dhall
 
 in
 
-let pipeline : DebianVersions.DebVersion -> PipelineMode.Type -> Pipeline.Config.Type = \(debVersion : DebianVersions.DebVersion) ->
-\(mode: PipelineMode.Type) -> 
+let pipeline : DebianVersions.DebVersion -> Profiles.Type ->  PipelineMode.Type -> Pipeline.Config.Type = 
+  \(debVersion : DebianVersions.DebVersion) ->
+  \(profile: Profiles.Type) ->
+  \(mode: PipelineMode.Type) -> 
     Pipeline.Config::{
       spec =
         JobSpec::{
           dirtyWhen = DebianVersions.dirtyWhen debVersion,
           path = "Release",
-          name = "MinaArtifact${DebianVersions.capitalName debVersion}",
+          name = "MinaArtifact${DebianVersions.capitalName debVersion}${Profiles.toSuffixUppercase profile}",
           tags = [ PipelineTag.Type.Long, PipelineTag.Type.Release ],
           mode = mode
         },
       steps = [
         Libp2p.step debVersion,
         Command.build
           Command.Config::{
-            commands = 
-              DebianVersions.toolchainRunner debVersion [
-                "DUNE_PROFILE=devnet",
-                "AWS_ACCESS_KEY_ID",
-                "AWS_SECRET_ACCESS_KEY",
-                "MINA_BRANCH=$BUILDKITE_BRANCH",
-                "MINA_COMMIT_SHA1=$BUILDKITE_COMMIT",
-                "MINA_DEB_CODENAME=${DebianVersions.lowerName debVersion}"
-              ] "./buildkite/scripts/build-artifact.sh",
-            label = "Build Mina for ${DebianVersions.capitalName debVersion}",
+            commands = DebianVersions.toolchainRunner debVersion [
+              "DUNE_PROFILE=${Profiles.duneProfile profile}",
+              "AWS_ACCESS_KEY_ID",
+              "AWS_SECRET_ACCESS_KEY",
+              "MINA_BRANCH=$BUILDKITE_BRANCH",
+              "MINA_COMMIT_SHA1=$BUILDKITE_COMMIT",
+              "MINA_DEB_CODENAME=${DebianVersions.lowerName debVersion}"
+            ] "./buildkite/scripts/build-artifact.sh",
+            label = "Build Mina for ${DebianVersions.capitalName debVersion} ${Profiles.toSuffixUppercase profile}",
             key = "build-deb-pkg",
             target = Size.XLarge,
             retries = [
@@ -55,10 +57,11 @@ let pipeline : DebianVersions.DebVersion -> PipelineMode.Type -> Pipeline.Config
 
         -- daemon berkeley image
         let daemonBerkeleySpec = DockerImage.ReleaseSpec::{
-          deps=DebianVersions.dependsOn debVersion,
+          deps=DebianVersions.dependsOn debVersion profile,
           service="mina-daemon",
           network="berkeley",
           deb_codename="${DebianVersions.lowerName debVersion}",
+          deb_profile="${Profiles.lowerName profile}",
           step_key="daemon-berkeley-${DebianVersions.lowerName debVersion}-docker-image"
         }
 
@@ -68,7 +71,7 @@ let pipeline : DebianVersions.DebVersion -> PipelineMode.Type -> Pipeline.Config
 
         -- test_executive image
         let testExecutiveSpec = DockerImage.ReleaseSpec::{
-          deps=DebianVersions.dependsOn debVersion,
+          deps=DebianVersions.dependsOn debVersion profile,
           service="mina-test-executive",
           deb_codename="${DebianVersions.lowerName debVersion}",
           step_key="test-executive-${DebianVersions.lowerName debVersion}-docker-image"
@@ -78,7 +81,7 @@ let pipeline : DebianVersions.DebVersion -> PipelineMode.Type -> Pipeline.Config
 
         -- batch_txn_tool image
         let batchTxnSpec = DockerImage.ReleaseSpec::{
-          deps=DebianVersions.dependsOn debVersion,
+          deps=DebianVersions.dependsOn debVersion profile,
           service="mina-batch-txn",
           network="berkeley",
           deb_codename="${DebianVersions.lowerName debVersion}",
@@ -89,29 +92,29 @@ let pipeline : DebianVersions.DebVersion -> PipelineMode.Type -> Pipeline.Config
 
         -- archive image
         let archiveSpec = DockerImage.ReleaseSpec::{
-          deps=DebianVersions.dependsOn debVersion,
+          deps=DebianVersions.dependsOn debVersion profile,
           service="mina-archive",
           deb_codename="${DebianVersions.lowerName debVersion}",
+          deb_profile="${Profiles.lowerName profile}",
           step_key="archive-${DebianVersions.lowerName debVersion}-docker-image"
         }
         in
         DockerImage.generateStep archiveSpec,
 
         -- rosetta image
         let rosettaSpec = DockerImage.ReleaseSpec::{
-          deps=DebianVersions.dependsOn debVersion,
+          deps=DebianVersions.dependsOn debVersion profile,
           service="mina-rosetta",
           network="berkeley",
           deb_codename="${DebianVersions.lowerName debVersion}",
           step_key="rosetta-${DebianVersions.lowerName debVersion}-docker-image"
         }
         in
-
         DockerImage.generateStep rosettaSpec,
 
         -- ZkApp test transaction image
         let zkappTestTxnSpec = DockerImage.ReleaseSpec::{
-          deps=DebianVersions.dependsOn debVersion,
+          deps=DebianVersions.dependsOn debVersion profile,
           service="mina-zkapp-test-transaction",
           deb_codename="${DebianVersions.lowerName debVersion}",
           step_key="zkapp-test-transaction-${DebianVersions.lowerName debVersion}-docker-image"
@@ -126,7 +129,8 @@ let pipeline : DebianVersions.DebVersion -> PipelineMode.Type -> Pipeline.Config
 
 in
 {
-  bullseye  = pipeline DebianVersions.DebVersion.Bullseye PipelineMode.Type.PullRequest
-  , buster  = pipeline DebianVersions.DebVersion.Buster PipelineMode.Type.PullRequest
-  , focal   = pipeline DebianVersions.DebVersion.Focal PipelineMode.Type.PullRequest
-}
+  bullseye  = pipeline DebianVersions.DebVersion.Bullseye Profiles.Type.Standard PipelineMode.Type.PullRequest
+  , bullseye-lightnet  = pipeline DebianVersions.DebVersion.Bullseye Profiles.Type.Lightnet PipelineMode.Type.PullRequest
+  , buster  = pipeline DebianVersions.DebVersion.Buster Profiles.Type.Standard PipelineMode.Type.PullRequest
+  , focal   = pipeline DebianVersions.DebVersion.Focal Profiles.Type.Standard PipelineMode.Type.PullRequest
+  }

buildkite/src/Constants/DebianVersions.dhall

@@ -1,6 +1,7 @@
 let Prelude = ../External/Prelude.dhall
 let RunInToolchain = ../Command/RunInToolchain.dhall
 let ContainerImages = ./ContainerImages.dhall
+let Profiles = ./Profiles.dhall
 let S = ../Lib/SelectFiles.dhall
 let D = S.PathPattern
 
@@ -46,13 +47,15 @@ let toolchainImage = \(debVersion : DebVersion) ->
     , Focal = ContainerImages.minaToolchainBullseye
   } debVersion
 
-let dependsOn = \(debVersion : DebVersion) ->
+let dependsOn = \(debVersion : DebVersion) -> \(profile : Profiles.Type) ->
+  let profileSuffix = Profiles.toSuffixUppercase profile in
+  let prefix = "MinaArtifact" in
   merge {
-    Bookworm = [{ name = "MinaArtifactBookworm", key = "build-deb-pkg" }]
-    , Bullseye = [{ name = "MinaArtifactBullseye", key = "build-deb-pkg" }]
-    , Buster = [{ name = "MinaArtifactBuster", key = "build-deb-pkg" }]
-    , Jammy = [{ name = "MinaArtifactJammy", key = "build-deb-pkg" }]
-    , Focal = [{ name = "MinaArtifactFocal", key = "build-deb-pkg" }]
+    Bookworm = [{ name = "${prefix}${profileSuffix}", key = "build-deb-pkg" }]
+    , Bullseye = [{ name = "${prefix}${capitalName debVersion}${profileSuffix}", key = "build-deb-pkg" }]
+    , Buster = [{ name = "${prefix}${capitalName debVersion}${profileSuffix}", key = "build-deb-pkg" }]
+    , Jammy = [{ name = "${prefix}${capitalName debVersion}${profileSuffix}", key = "build-deb-pkg" }]
+    , Focal = [{ name = "${prefix}${capitalName debVersion}${profileSuffix}", key = "build-deb-pkg" }]
   } debVersion
 
 -- Most debian builds are only used for public releases
@@ -105,4 +108,4 @@ in
   , toolchainImage = toolchainImage
   , dependsOn = dependsOn
   , dirtyWhen = dirtyWhen
-}
+}

buildkite/src/Constants/DockerVersions.dhall

@@ -0,0 +1,43 @@
+let Prelude = ../External/Prelude.dhall
+let Profiles = ./Profiles.dhall
+
+let Docker: Type  = < Bookworm | Bullseye | Buster | Jammy | Focal >
+
+let capitalName = \(docker : Docker) ->
+  merge {
+    Bookworm = "Bookworm"
+    , Bullseye = "Bullseye"
+    , Buster = "Buster"
+    , Jammy = "Jammy"
+    , Focal = "Focal"
+  } docker
+
+let lowerName = \(docker : Docker) ->
+  merge {
+    Bookworm = "bookworm"
+    , Bullseye = "bullseye"
+    , Buster = "buster"
+    , Jammy = "jammy"
+    , Focal = "focal"
+  } docker
+
+let dependsOn = \(docker : Docker) -> \(profile : Profiles.Type) -> \(binary: Text) -> 
+  let profileSuffix = Profiles.toSuffixUppercase profile in
+  let prefix = "MinaArtifact" in 
+  let suffix = "docker-image" in
+  merge {
+    Bookworm = [{ name = "${prefix}${profileSuffix}", key = "${binary}-${lowerName docker}-${suffix}" }]
+    , Bullseye = [{ name = "${prefix}${capitalName docker}${profileSuffix}", key = "${binary}-${lowerName docker}-${suffix}" }]
+    , Buster = [{ name = "${prefix}${capitalName docker}${profileSuffix}", key = "${binary}-${lowerName docker}-${suffix}" }]
+    , Jammy = [{ name = "${prefix}${capitalName docker}${profileSuffix}", key = "${binary}-${lowerName docker}-${suffix}" }]
+    , Focal = [{ name = "${prefix}${capitalName docker}${profileSuffix}", key = "${binary}-${lowerName docker}-${suffix}" }]
+  } docker
+
+in
+
+{
+  Type = Docker
+  , capitalName = capitalName
+  , lowerName = lowerName
+  , dependsOn = dependsOn
+}

buildkite/src/Constants/Profiles.dhall

@@ -0,0 +1,46 @@
+let Prelude = ../External/Prelude.dhall
+
+let Profile : Type = < Standard | Lightnet >
+
+let capitalName = \(profile : Profile) ->
+  merge {
+    Standard = "Standard"
+    , Lightnet = "Lightnet"
+  } profile
+
+let lowerName = \(profile : Profile) ->
+  merge {
+    Standard = "standard"
+    , Lightnet = "lightnet"
+  } profile
+
+let duneProfile = \(profile : Profile) ->
+  merge {
+    Standard = "devnet"
+    , Lightnet = "lightnet"
+  } profile
+
+let toSuffixUppercase = \(profile : Profile) ->
+  merge {
+    Standard = ""
+    , Lightnet = "Lightnet"
+  } profile
+
+let toSuffixLowercase = \(profile : Profile) ->
+  merge {
+    Standard = ""
+    , Lightnet = "lightnet"
+  } profile
+
+
+
+in
+
+{
+  Type = Profile
+  , capitalName = capitalName
+  , lowerName = lowerName
+  , duneProfile = duneProfile
+  , toSuffixUppercase = toSuffixUppercase
+  , toSuffixLowercase = toSuffixLowercase
+}

buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall

@@ -12,12 +12,14 @@ let JobSpec = ../../Pipeline/JobSpec.dhall
 let Command = ../../Command/Base.dhall
 let Size = ../../Command/Size.dhall
 let DockerImage = ../../Command/DockerImage.dhall
+let Profiles = ../../Constants/Profiles.dhall
+let DebianVersions = ../../Constants/DebianVersions.dhall
 
 let spec = DockerImage.ReleaseSpec::{
     service="itn-orchestrator",
     step_key="itn-orchestrator-docker-image",
     network="berkeley",
-    deps = [ { name = "MinaArtifactBullseye", key = "daemon-berkeley-bullseye-docker-image" } ]
+    deps = DebianVersions.dependsOn DebianVersions.DebVersion.Bullseye Profiles.Type.Standard
 }
 
 in	

buildkite/src/Jobs/Release/MinaArtifactBullseyeLightnet.dhall

@@ -0,0 +1,6 @@
+let ArtifactPipelines = ../../Command/MinaArtifact.dhall
+let Pipeline = ../../Pipeline/Dsl.dhall
+
+in
+
+Pipeline.build ArtifactPipelines.bullseye-lightnet

buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall

@@ -5,10 +5,11 @@ let Pipeline = ../../Pipeline/Dsl.dhall
 let PipelineTag = ../../Pipeline/Tag.dhall
 
 let CheckGraphQLSchema = ../../Command/CheckGraphQLSchema.dhall
+let DebianVersions = ../../Constants/DebianVersions.dhall
+let Profiles = ../../Constants/Profiles.dhall
+
+let dependsOn = DebianVersions.dependsOn DebianVersions.DebVersion.Bullseye Profiles.Type.Standard
 
-let dependsOn = [
-    { name = "MinaArtifactBullseye", key = "build-deb-pkg" }
-]
 
 in Pipeline.build Pipeline.Config::{
   spec =

buildkite/src/Jobs/Test/ConnectToBerkeley.dhall

@@ -5,10 +5,10 @@ let Pipeline = ../../Pipeline/Dsl.dhall
 let PipelineTag = ../../Pipeline/Tag.dhall
 
 let ConnectToTestnet = ../../Command/ConnectToTestnet.dhall
+let Profiles = ../../Constants/Profiles.dhall
+let Dockers = ../../Constants/DockerVersions.dhall
 
-let dependsOn = [
-  { name = "MinaArtifactBullseye", key = "daemon-berkeley-bullseye-docker-image" }
-]
+let dependsOn = Dockers.dependsOn Dockers.Type.Bullseye Profiles.Type.Standard "daemon-berkeley"
 
 in Pipeline.build Pipeline.Config::{
   spec =

buildkite/src/Jobs/Test/ReplayerTest.dhall.SUSPENDED

@@ -5,9 +5,10 @@ let Pipeline = ../../Pipeline/Dsl.dhall
 let JobSpec = ../../Pipeline/JobSpec.dhall
 
 let ReplayerTest = ../../Command/ReplayerTest.dhall
+let Profiles = ../../Constants/Profiles.dhall
+let Dockers = ../../Constants/DockerVersions.dhall
 
-let dependsOn =
-      [ { name = "MinaArtifactBullseye", key = "archive-bullseye-docker-image" } ]
+let dependsOn = Dockers.dependsOn Dockers.Type.Bullseye Profiles.Type.Standard "archive"
 
 in  Pipeline.build
       Pipeline.Config::{

buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall

@@ -13,7 +13,8 @@ let RunInToolchain = ../../Command/RunInToolchain.dhall
 let Size = ../../Command/Size.dhall
 let Libp2p = ../../Command/Libp2pHelperBuild.dhall
 let DockerImage = ../../Command/DockerImage.dhall
-let DebianVersions = ../../Constants/DebianVersions.dhall
+let Profiles = ../../Constants/Profiles.dhall
+let Dockers = ../../Constants/DockerVersions.dhall
 
 let dirtyWhen = [ 
   S.strictlyStart (S.contains "src/app/rosetta"),
@@ -47,7 +48,7 @@ Pipeline.build
           label = "Rosetta integration tests Bullseye"
           , key = "rosetta-integration-tests-bullseye"
           , target = Size.Small
-          , depends_on = [ { name = "MinaArtifactBullseye", key = "rosetta-bullseye-docker-image" } ]
+          , depends_on = Dockers.dependsOn Dockers.Type.Bullseye Profiles.Type.Standard "rosetta"
         }
     ]
   }

buildkite/src/Jobs/Test/RunSnarkProfiler.dhall

@@ -8,15 +8,16 @@ let Pipeline = ../../Pipeline/Dsl.dhall
 let PipelineTag = ../../Pipeline/Tag.dhall
 let JobSpec = ../../Pipeline/JobSpec.dhall
 
+let DebianVersions = ../../Constants/DebianVersions.dhall
+let Profiles = ../../Constants/Profiles.dhall
+
 let Command = ../../Command/Base.dhall
 let RunInToolchain = ../../Command/RunInToolchain.dhall
 let Docker = ../../Command/Docker/Type.dhall
 let Size = ../../Command/Size.dhall
 
 
-let dependsOn = [
-    { name = "MinaArtifactBullseye", key = "build-deb-pkg" }
-]
+let dependsOn = DebianVersions.dependsOn DebianVersions.DebVersion.Bullseye Profiles.Type.Standard
 
 in