Divide CI into stages

MinaProtocol · Sep 6, 2023 · ca7280e · ca7280e
1 parent 862a150
commit ca7280e
Show file tree

Hide file tree

Showing 19 changed files with 236 additions and 66 deletions.
diff --git a/buildkite/src/Command/MinaArtifact.dhall b/buildkite/src/Command/MinaArtifact.dhall
@@ -5,6 +5,9 @@ let S = ../Lib/SelectFiles.dhall
 let D = S.PathPattern
 
 let Pipeline = ../Pipeline/Dsl.dhall
+let PipelineStage = ../Pipeline/Stage.dhall
+let PipelineMode = ../Pipeline/Mode.dhall
+
 let JobSpec = ../Pipeline/JobSpec.dhall
 
 let Command = ./Base.dhall
@@ -16,13 +19,16 @@ let DebianVersions = ../Constants/DebianVersions.dhall
 
 in
 
-let pipeline : DebianVersions.DebVersion -> Pipeline.Config.Type = \(debVersion : DebianVersions.DebVersion) ->
+let pipeline : DebianVersions.DebVersion -> PipelineMode.Type -> Pipeline.Config.Type = \(debVersion : DebianVersions.DebVersion) ->
+  \(mode: PipelineMode.Type) -> 
     Pipeline.Config::{
       spec =
         JobSpec::{
           dirtyWhen = DebianVersions.dirtyWhen debVersion,
           path = "Release",
-          name = "MinaArtifact${DebianVersions.capitalName debVersion}"
+          name = "MinaArtifact${DebianVersions.capitalName debVersion}",
+          stage = PipelineStage.Type.Stage2,
+          mode = mode
         },
       steps = [
         Libp2p.step debVersion,
@@ -107,7 +113,7 @@ let pipeline : DebianVersions.DebVersion -> Pipeline.Config.Type = \(debVersion
 
 in
 {
-  bullseye  = pipeline DebianVersions.DebVersion.Bullseye
-  , buster  = pipeline DebianVersions.DebVersion.Buster
-  , focal   = pipeline DebianVersions.DebVersion.Focal
+  bullseye  = pipeline DebianVersions.DebVersion.Bullseye PipelineMode.Type.PullRequest
+  , buster  = pipeline DebianVersions.DebVersion.Buster PipelineMode.Type.Stable
+  , focal   = pipeline DebianVersions.DebVersion.Focal PipelineMode.Type.Stable
 }
diff --git a/buildkite/src/Jobs/Lint/Fast.dhall b/buildkite/src/Jobs/Lint/Fast.dhall
@@ -5,6 +5,7 @@ let B = ../../External/Buildkite.dhall
 let S = ../../Lib/SelectFiles.dhall
 
 let Pipeline = ../../Pipeline/Dsl.dhall
+let PipelineStage = ../../Pipeline/Stage.dhall
 
 let JobSpec = ../../Pipeline/JobSpec.dhall
 
@@ -18,6 +19,8 @@ let RunInToolchain = ../../Command/RunInToolchain.dhall
 
 let Size = ../../Command/Size.dhall
 
+let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type
+
 let commands =
       [ Cmd.run "./scripts/lint_codeowners.sh"
       , Cmd.run "./scripts/lint_rfcs.sh"
@@ -47,6 +50,7 @@ in  Pipeline.build
                 "Fast lint steps; CODEOWNERs, RFCs, Check Snarky & Proof-Systems submodules, Preprocessor Deps"
             , key = "lint"
             , target = Size.Small
+            , soft_fail = Some (B/SoftFail.Boolean True)
             , docker = Some Docker::{
               , image = (../../Constants/ContainerImages.dhall).toolchainBase
               }

diff --git a/buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall b/buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall
@@ -0,0 +1,51 @@
+let Prelude = ../../External/Prelude.dhall
+
+let Cmd = ../../Lib/Cmds.dhall
+let S = ../../Lib/SelectFiles.dhall
+
+let Pipeline = ../../Pipeline/Dsl.dhall
+let PipelineStage = ../../Pipeline/Stage.dhall
+let JobSpec = ../../Pipeline/JobSpec.dhall
+
+let Command = ../../Command/Base.dhall
+let Size = ../../Command/Size.dhall
+let DockerImage = ../../Command/DockerImage.dhall
+let DockerLogin = ../../Command/DockerLogin/Type.dhall
+
+
+in
+
+Pipeline.build
+  Pipeline.Config::{
+    spec =
+      JobSpec::{
+        dirtyWhen = [
+          S.strictlyStart (S.contains "dockerfiles/stages/1-"),
+          S.strictlyStart (S.contains "dockerfiles/stages/2-"),
+          S.strictlyStart (S.contains "dockerfiles/stages/3-"),
+          S.strictlyStart (S.contains "buildkite/src/Jobs/Release/MinaToolchainArtifact"),
+          S.strictly (S.contains "opam.export"),
+          -- Rust version has changed
+          S.strictlyEnd (S.contains "rust-toolchain.toml")
+        ],
+        path = "Release",
+        name = "MinaToolchainArtifactBullseye",
+        stage = PipelineStage.Type.Stage2
+
+      },
+    steps = [
+
+      -- mina-toolchain Debian 11 "Bullseye" Toolchain
+      let toolchainBullseyeSpec = DockerImage.ReleaseSpec::{
+        service="mina-toolchain",
+        deb_codename="bullseye",
+        extra_args="--no-cache",
+        step_key="toolchain-bullseye-docker-image"
+      }
+
+      in
+
+      DockerImage.generateStep toolchainBullseyeSpec
+
+    ]
+  }
diff --git a/.../Jobs/Release/MinaToolchainArtifact.dhall → ...Release/MinaToolchainArtifactBuster.dhall b/.../Jobs/Release/MinaToolchainArtifact.dhall → ...Release/MinaToolchainArtifactBuster.dhall
@@ -4,6 +4,8 @@ let Cmd = ../../Lib/Cmds.dhall
 let S = ../../Lib/SelectFiles.dhall
 
 let Pipeline = ../../Pipeline/Dsl.dhall
+let PipelineStage = ../../Pipeline/Stage.dhall
+let PipelineMode = ../../Pipeline/Mode.dhall
 let JobSpec = ../../Pipeline/JobSpec.dhall
 
 let Command = ../../Command/Base.dhall
@@ -28,22 +30,12 @@ Pipeline.build
           S.strictlyEnd (S.contains "rust-toolchain.toml")
         ],
         path = "Release",
-        name = "MinaToolchainArtifact"
+        name = "MinaToolchainArtifactBuster",
+        stage = PipelineStage.Type.Stage2,
+        mode = PipelineMode.Type.Stable        
       },
     steps = [
 
-      -- mina-toolchain Debian 11 "Bullseye" Toolchain
-      let toolchainBullseyeSpec = DockerImage.ReleaseSpec::{
-        service="mina-toolchain",
-        deb_codename="bullseye",
-        extra_args="--no-cache",
-        step_key="toolchain-bullseye-docker-image"
-      }
-
-      in
-
-      DockerImage.generateStep toolchainBullseyeSpec,
-
       -- mina-toolchain Debian 10 "Buster" Toolchain
       let toolchainBusterSpec = DockerImage.ReleaseSpec::{
         service="mina-toolchain",

diff --git a/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall b/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall
@@ -2,6 +2,7 @@ let S = ../../Lib/SelectFiles.dhall
 
 let JobSpec = ../../Pipeline/JobSpec.dhall
 let Pipeline = ../../Pipeline/Dsl.dhall
+let PipelineStage = ../../Pipeline/Stage.dhall
 
 let CheckGraphQLSchema = ../../Command/CheckGraphQLSchema.dhall
 
@@ -18,7 +19,8 @@ in Pipeline.build Pipeline.Config::{
       S.strictly (S.contains "Makefile")
     ],
     path = "Test",
-    name = "CheckGraphQLSchema"
+    name = "CheckGraphQLSchema",
+    stage = PipelineStage.Type.Stage2
   },
   steps = [
     CheckGraphQLSchema.step dependsOn

diff --git a/buildkite/src/Jobs/Test/ConnectToBerkeley.dhall b/buildkite/src/Jobs/Test/ConnectToBerkeley.dhall
@@ -2,6 +2,7 @@ let S = ../../Lib/SelectFiles.dhall
 
 let JobSpec = ../../Pipeline/JobSpec.dhall
 let Pipeline = ../../Pipeline/Dsl.dhall
+let PipelineStage = ../../Pipeline/Stage.dhall
 
 let ConnectToTestnet = ../../Command/ConnectToTestnet.dhall
 
@@ -19,7 +20,8 @@ in Pipeline.build Pipeline.Config::{
       S.exactly "buildkite/src/Command/ConnectToTestnet" "dhall"
     ],
     path = "Test",
-    name = "ConnectToBerkeley"
+    name = "ConnectToBerkeley",
+    stage = PipelineStage.Type.Stage2
   },
   steps = [
     ConnectToTestnet.step dependsOn

diff --git a/buildkite/src/Jobs/Test/CoverageTearDown.dhall b/buildkite/src/Jobs/Test/CoverageTearDown.dhall
@@ -0,0 +1,33 @@
+let Prelude =  ../../External/Prelude.dhall
+let S = ../../Lib/SelectFiles.dhall
+let Cmd = ../../Lib/Cmds.dhall
+let RunInToolchain = ../../Command/RunInToolchain.dhall
+let Command = ../../Command/Base.dhall
+let Docker = ../../Command/Docker/Type.dhall
+let Size = ../../Command/Size.dhall
+let JobSpec = ../../Pipeline/JobSpec.dhall
+let Pipeline = ../../Pipeline/Dsl.dhall
+let PipelineMode = ../../Pipeline/Mode.dhall
+let PipelineStage = ../../Pipeline/Stage.dhall
+
+in Pipeline.build 
+  Pipeline.Config::{
+    spec =
+      JobSpec::{
+        dirtyWhen = [ S.everything ],
+        path = "Test",
+        stage = PipelineStage.Type.TearDown,
+        name = "CoverageTearDown"
+    }
+  , steps = [
+      Command.build
+        Command.Config::{
+          commands =
+            RunInToolchain.runInToolchainBullseye ["COVERALLS_TOKEN"]
+              "buildkite/scripts/finish-coverage-data-upload.sh",
+          label = "Finish coverage data gathering",
+          key = "finish-coverage-data",
+          target = Size.Small
+        }
+    ]
+  }
diff --git a/buildkite/src/Jobs/Test/DaemonUnitTest.dhall b/buildkite/src/Jobs/Test/DaemonUnitTest.dhall
@@ -5,6 +5,8 @@ let S = ../../Lib/SelectFiles.dhall
 let D = S.PathPattern
 
 let Pipeline = ../../Pipeline/Dsl.dhall
+let PipelineStage = ../../Pipeline/Stage.dhall
+
 let JobSpec = ../../Pipeline/JobSpec.dhall
 
 let Command = ../../Command/Base.dhall
@@ -44,7 +46,8 @@ Pipeline.build
       JobSpec::{
         dirtyWhen = unitDirtyWhen,
         path = "Test",
-        name = "DaemonUnitTest"
+        name = "DaemonUnitTest",
+        stage = PipelineStage.Type.Stage2
       },
     steps = [
       buildTestCmd "dev" "src/lib" Size.XLarge

diff --git a/buildkite/src/Jobs/Test/DelegationBackendUnitTest.dhall b/buildkite/src/Jobs/Test/DelegationBackendUnitTest.dhall
@@ -4,6 +4,8 @@ let Cmd = ../../Lib/Cmds.dhall
 let S = ../../Lib/SelectFiles.dhall
 
 let Pipeline = ../../Pipeline/Dsl.dhall
+let PipelineStage = ../../Pipeline/Stage.dhall
+
 let JobSpec = ../../Pipeline/JobSpec.dhall
 
 let Command = ../../Command/Base.dhall
@@ -22,7 +24,8 @@ Pipeline.build
           S.exactly "buildkite/src/Jobs/Test/DelegationBackendUnitTest" "dhall"
         ],
         path = "Test",
-        name = "DelegationBackendUnitTest"
+        name = "DelegationBackendUnitTest",
+        stage = PipelineStage.Type.Stage1
       },
     steps = [
       Command.build

diff --git a/buildkite/src/Jobs/Test/Libp2pUnitTest.dhall b/buildkite/src/Jobs/Test/Libp2pUnitTest.dhall
@@ -4,6 +4,7 @@ let Cmd = ../../Lib/Cmds.dhall
 let S = ../../Lib/SelectFiles.dhall
 
 let Pipeline = ../../Pipeline/Dsl.dhall
+let PipelineStage = ../../Pipeline/Stage.dhall
 let JobSpec = ../../Pipeline/JobSpec.dhall
 
 let Command = ../../Command/Base.dhall
@@ -23,7 +24,8 @@ Pipeline.build
           S.exactly "buildkite/src/Jobs/Test/Libp2pUnitTest" "dhall"
         ],
         path = "Test",
-        name = "Libp2pUnitTest"
+        name = "Libp2pUnitTest",
+        stage = PipelineStage.Type.Stage2
       },
     steps = [
       Command.build

diff --git a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall
@@ -5,6 +5,7 @@ let Cmd = ../../Lib/Cmds.dhall
 let S = ../../Lib/SelectFiles.dhall
 
 let Pipeline = ../../Pipeline/Dsl.dhall
+let PipelineStage = ../../Pipeline/Stage.dhall
 let JobSpec = ../../Pipeline/JobSpec.dhall
 
 let Command = ../../Command/Base.dhall
@@ -28,7 +29,8 @@ Pipeline.build
       JobSpec::{
         dirtyWhen = dirtyWhen,
         path = "Test",
-        name = "RosettaIntegrationTests"
+        name = "RosettaIntegrationTests",
+        stage = PipelineStage.Type.Stage2
       }
     , steps = [
       Command.build

diff --git a/buildkite/src/Jobs/Test/RunSnarkProfiler.dhall b/buildkite/src/Jobs/Test/RunSnarkProfiler.dhall
@@ -5,6 +5,7 @@ let S = ../../Lib/SelectFiles.dhall
 let D = S.PathPattern
 
 let Pipeline = ../../Pipeline/Dsl.dhall
+let PipelineStage = ../../Pipeline/Stage.dhall
 let JobSpec = ../../Pipeline/JobSpec.dhall
 
 let Command = ../../Command/Base.dhall
@@ -54,7 +55,8 @@ Pipeline.build
       JobSpec::{
         dirtyWhen = lintDirtyWhen,
         path = "Test",
-        name = "RunSnarkProfiler"
+        name = "RunSnarkProfiler",
+        stage = PipelineStage.Type.Stage2
       },
     steps = [
       buildTestCmd Size.Small dependsOn

diff --git a/buildkite/src/Jobs/Test/SingleNodeTest.dhall b/buildkite/src/Jobs/Test/SingleNodeTest.dhall
@@ -5,6 +5,8 @@ let S = ../../Lib/SelectFiles.dhall
 let D = S.PathPattern
 
 let Pipeline = ../../Pipeline/Dsl.dhall
+let PipelineStage = ../../Pipeline/Stage.dhall
+
 let JobSpec = ../../Pipeline/JobSpec.dhall
 
 let Command = ../../Command/Base.dhall
@@ -56,7 +58,8 @@ Pipeline.build
       JobSpec::{
         dirtyWhen = unitDirtyWhen,
         path = "Test",
-        name = "SingleNodeTest"
+        name = "SingleNodeTest",
+        stage = PipelineStage.Type.Stage2
       },
     steps = [
       buildTestCmd "dev" "src/test/command_line_tests/command_line_tests.exe" Size.XLarge

diff --git a/buildkite/src/Jobs/Test/SnarkyJSTest.dhall b/buildkite/src/Jobs/Test/SnarkyJSTest.dhall
@@ -2,6 +2,7 @@ let S = ../../Lib/SelectFiles.dhall
 let B = ../../External/Buildkite.dhall
 
 let Pipeline = ../../Pipeline/Dsl.dhall
+let PipelineStage = ../../Pipeline/Stage.dhall
 let JobSpec = ../../Pipeline/JobSpec.dhall
 
 let Command = ../../Command/Base.dhall
@@ -24,7 +25,8 @@ Pipeline.build
           S.strictlyStart (S.contains "src/lib")
         ],
         path = "Test",
-        name = "SnarkyJSTest"
+        name = "SnarkyJSTest",
+        stage = PipelineStage.Type.Stage2
       },
     steps = [
       Command.build

diff --git a/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall b/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall
@@ -2,6 +2,8 @@ let S = ../../Lib/SelectFiles.dhall
 
 let JobSpec = ../../Pipeline/JobSpec.dhall
 let Pipeline = ../../Pipeline/Dsl.dhall
+let PipelineMode = ../../Pipeline/Mode.dhall
+let PipelineStage = ../../Pipeline/Stage.dhall
 
 let TestExecutive = ../../Command/TestExecutive.dhall
 
@@ -29,7 +31,8 @@ in Pipeline.build Pipeline.Config::{
         S.strictlyStart (S.contains "automation/terraform/modules/kubernetes/testnet")
     ],
     path = "Test",
-    name = "TestnetIntegrationTests"
+    name = "TestnetIntegrationTests",
+    stage = PipelineStage.Type.Stage2
   },
   steps = [
     TestExecutive.build "integration_tests",