Skip to content

Commit 67753c3

Browse files
dkijaniadkijania
committed
Implemented 2 new stages in CI Fast and Long to better jobs organization and to improve feedback speed
1 parent 152303f commit 67753c3

19 files changed

+119
-37
lines changed

‎buildkite/src/Command/MinaArtifact.dhall

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,9 @@ let S = ../Lib/SelectFiles.dhall
55
let D = S.PathPattern
66

77
let Pipeline = ../Pipeline/Dsl.dhall
8+
let PipelineStage = ../Pipeline/Stage.dhall
9+
let PipelineMode = ../Pipeline/Mode.dhall
10+
811
let JobSpec = ../Pipeline/JobSpec.dhall
912

1013
let Command = ./Base.dhall
@@ -16,13 +19,16 @@ let DebianVersions = ../Constants/DebianVersions.dhall
1619

1720
in
1821

19-
let pipeline : DebianVersions.DebVersion -> Pipeline.Config.Type = \(debVersion : DebianVersions.DebVersion) ->
22+
let pipeline : DebianVersions.DebVersion -> PipelineMode.Type -> Pipeline.Config.Type = \(debVersion : DebianVersions.DebVersion) ->
23+
\(mode: PipelineMode.Type) ->
2024
Pipeline.Config::{
2125
spec =
2226
JobSpec::{
2327
dirtyWhen = DebianVersions.dirtyWhen debVersion,
2428
path = "Release",
25-
name = "MinaArtifact${DebianVersions.capitalName debVersion}"
29+
name = "MinaArtifact${DebianVersions.capitalName debVersion}",
30+
stage = PipelineStage.Type.Long,
31+
mode = mode
2632
},
2733
steps = [
2834
Libp2p.step debVersion,
@@ -107,7 +113,7 @@ let pipeline : DebianVersions.DebVersion -> Pipeline.Config.Type = \(debVersion
107113

108114
in
109115
{
110-
bullseye = pipeline DebianVersions.DebVersion.Bullseye
111-
, buster = pipeline DebianVersions.DebVersion.Buster
112-
, focal = pipeline DebianVersions.DebVersion.Focal
116+
bullseye = pipeline DebianVersions.DebVersion.Bullseye PipelineMode.Type.PullRequest
117+
, buster = pipeline DebianVersions.DebVersion.Buster PipelineMode.Type.Stable
118+
, focal = pipeline DebianVersions.DebVersion.Focal PipelineMode.Type.Stable
113119
}

‎buildkite/src/Jobs/Lint/Fast.dhall

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ let B = ../../External/Buildkite.dhall
55
let S = ../../Lib/SelectFiles.dhall
66

77
let Pipeline = ../../Pipeline/Dsl.dhall
8+
let PipelineStage = ../../Pipeline/Stage.dhall
89

910
let JobSpec = ../../Pipeline/JobSpec.dhall
1011

@@ -18,6 +19,8 @@ let RunInToolchain = ../../Command/RunInToolchain.dhall
1819

1920
let Size = ../../Command/Size.dhall
2021

22+
let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type
23+
2124
let commands =
2225
[ Cmd.run "./scripts/lint_codeowners.sh"
2326
, Cmd.run "./scripts/lint_rfcs.sh"
@@ -47,6 +50,7 @@ in Pipeline.build
4750
"Fast lint steps; CODEOWNERs, RFCs, Check Snarky & Proof-Systems submodules, Preprocessor Deps"
4851
, key = "lint"
4952
, target = Size.Small
53+
, soft_fail = Some (B/SoftFail.Boolean True)
5054
, docker = Some Docker::{
5155
, image = (../../Constants/ContainerImages.dhall).toolchainBase
5256
}

‎buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
let Prelude = ../../External/Prelude.dhall
2+
3+
let Cmd = ../../Lib/Cmds.dhall
4+
let S = ../../Lib/SelectFiles.dhall
5+
6+
let Pipeline = ../../Pipeline/Dsl.dhall
7+
let PipelineStage = ../../Pipeline/Stage.dhall
8+
let JobSpec = ../../Pipeline/JobSpec.dhall
9+
10+
let Command = ../../Command/Base.dhall
11+
let Size = ../../Command/Size.dhall
12+
let DockerImage = ../../Command/DockerImage.dhall
13+
let DockerLogin = ../../Command/DockerLogin/Type.dhall
14+
15+
16+
in
17+
18+
Pipeline.build
19+
Pipeline.Config::{
20+
spec =
21+
JobSpec::{
22+
dirtyWhen = [
23+
S.strictlyStart (S.contains "dockerfiles/stages/1-"),
24+
S.strictlyStart (S.contains "dockerfiles/stages/2-"),
25+
S.strictlyStart (S.contains "dockerfiles/stages/3-"),
26+
S.strictlyStart (S.contains "buildkite/src/Jobs/Release/MinaToolchainArtifact"),
27+
S.strictly (S.contains "opam.export"),
28+
-- Rust version has changed
29+
S.strictlyEnd (S.contains "rust-toolchain.toml")
30+
],
31+
path = "Release",
32+
name = "MinaToolchainArtifactBullseye",
33+
stage = PipelineStage.Type.Long
34+
35+
},
36+
steps = [
37+
38+
-- mina-toolchain Debian 11 "Bullseye" Toolchain
39+
let toolchainBullseyeSpec = DockerImage.ReleaseSpec::{
40+
service="mina-toolchain",
41+
deb_codename="bullseye",
42+
extra_args="--no-cache",
43+
step_key="toolchain-bullseye-docker-image"
44+
}
45+
46+
in
47+
48+
DockerImage.generateStep toolchainBullseyeSpec
49+
50+
]
51+
}

‎buildkite/src/Jobs/Release/MinaToolchainArtifact.dhall renamed to ‎buildkite/src/Jobs/Release/MinaToolchainArtifactBuster.dhall

Lines changed: 5 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ let Cmd = ../../Lib/Cmds.dhall
44
let S = ../../Lib/SelectFiles.dhall
55

66
let Pipeline = ../../Pipeline/Dsl.dhall
7+
let PipelineStage = ../../Pipeline/Stage.dhall
8+
let PipelineMode = ../../Pipeline/Mode.dhall
79
let JobSpec = ../../Pipeline/JobSpec.dhall
810

911
let Command = ../../Command/Base.dhall
@@ -28,22 +30,12 @@ Pipeline.build
2830
S.strictlyEnd (S.contains "rust-toolchain.toml")
2931
],
3032
path = "Release",
31-
name = "MinaToolchainArtifact"
33+
name = "MinaToolchainArtifactBuster",
34+
stage = PipelineStage.Type.Long,
35+
mode = PipelineMode.Type.Stable
3236
},
3337
steps = [
3438

35-
-- mina-toolchain Debian 11 "Bullseye" Toolchain
36-
let toolchainBullseyeSpec = DockerImage.ReleaseSpec::{
37-
service="mina-toolchain",
38-
deb_codename="bullseye",
39-
extra_args="--no-cache",
40-
step_key="toolchain-bullseye-docker-image"
41-
}
42-
43-
in
44-
45-
DockerImage.generateStep toolchainBullseyeSpec,
46-
4739
-- mina-toolchain Debian 10 "Buster" Toolchain
4840
let toolchainBusterSpec = DockerImage.ReleaseSpec::{
4941
service="mina-toolchain",

‎buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ let S = ../../Lib/SelectFiles.dhall
22

33
let JobSpec = ../../Pipeline/JobSpec.dhall
44
let Pipeline = ../../Pipeline/Dsl.dhall
5+
let PipelineStage = ../../Pipeline/Stage.dhall
56

67
let CheckGraphQLSchema = ../../Command/CheckGraphQLSchema.dhall
78

@@ -18,7 +19,8 @@ in Pipeline.build Pipeline.Config::{
1819
S.strictly (S.contains "Makefile")
1920
],
2021
path = "Test",
21-
name = "CheckGraphQLSchema"
22+
name = "CheckGraphQLSchema",
23+
stage = PipelineStage.Type.Long
2224
},
2325
steps = [
2426
CheckGraphQLSchema.step dependsOn

‎buildkite/src/Jobs/Test/ConnectToBerkeley.dhall

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ let S = ../../Lib/SelectFiles.dhall
22

33
let JobSpec = ../../Pipeline/JobSpec.dhall
44
let Pipeline = ../../Pipeline/Dsl.dhall
5+
let PipelineStage = ../../Pipeline/Stage.dhall
56

67
let ConnectToTestnet = ../../Command/ConnectToTestnet.dhall
78

@@ -19,7 +20,8 @@ in Pipeline.build Pipeline.Config::{
1920
S.exactly "buildkite/src/Command/ConnectToTestnet" "dhall"
2021
],
2122
path = "Test",
22-
name = "ConnectToBerkeley"
23+
name = "ConnectToBerkeley",
24+
stage = PipelineStage.Type.Long
2325
},
2426
steps = [
2527
ConnectToTestnet.step dependsOn

‎buildkite/src/Jobs/Test/DaemonUnitTest.dhall

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ let S = ../../Lib/SelectFiles.dhall
55
let D = S.PathPattern
66

77
let Pipeline = ../../Pipeline/Dsl.dhall
8+
let PipelineStage = ../../Pipeline/Stage.dhall
9+
810
let JobSpec = ../../Pipeline/JobSpec.dhall
911

1012
let Command = ../../Command/Base.dhall
@@ -44,7 +46,8 @@ Pipeline.build
4446
JobSpec::{
4547
dirtyWhen = unitDirtyWhen,
4648
path = "Test",
47-
name = "DaemonUnitTest"
49+
name = "DaemonUnitTest",
50+
stage = PipelineStage.Type.Long
4851
},
4952
steps = [
5053
buildTestCmd "dev" "src/lib" Size.XLarge

‎buildkite/src/Jobs/Test/DelegationBackendUnitTest.dhall

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ let Cmd = ../../Lib/Cmds.dhall
44
let S = ../../Lib/SelectFiles.dhall
55

66
let Pipeline = ../../Pipeline/Dsl.dhall
7+
let PipelineStage = ../../Pipeline/Stage.dhall
8+
79
let JobSpec = ../../Pipeline/JobSpec.dhall
810

911
let Command = ../../Command/Base.dhall
@@ -22,7 +24,8 @@ Pipeline.build
2224
S.exactly "buildkite/src/Jobs/Test/DelegationBackendUnitTest" "dhall"
2325
],
2426
path = "Test",
25-
name = "DelegationBackendUnitTest"
27+
name = "DelegationBackendUnitTest",
28+
stage = PipelineStage.Type.Fast
2629
},
2730
steps = [
2831
Command.build

‎buildkite/src/Jobs/Test/Libp2pUnitTest.dhall

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ let Cmd = ../../Lib/Cmds.dhall
44
let S = ../../Lib/SelectFiles.dhall
55

66
let Pipeline = ../../Pipeline/Dsl.dhall
7+
let PipelineStage = ../../Pipeline/Stage.dhall
78
let JobSpec = ../../Pipeline/JobSpec.dhall
89

910
let Command = ../../Command/Base.dhall
@@ -23,7 +24,8 @@ Pipeline.build
2324
S.exactly "buildkite/src/Jobs/Test/Libp2pUnitTest" "dhall"
2425
],
2526
path = "Test",
26-
name = "Libp2pUnitTest"
27+
name = "Libp2pUnitTest",
28+
stage = PipelineStage.Type.Long
2729
},
2830
steps = [
2931
Command.build

‎buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ let Cmd = ../../Lib/Cmds.dhall
55
let S = ../../Lib/SelectFiles.dhall
66

77
let Pipeline = ../../Pipeline/Dsl.dhall
8+
let PipelineStage = ../../Pipeline/Stage.dhall
89
let JobSpec = ../../Pipeline/JobSpec.dhall
910

1011
let Command = ../../Command/Base.dhall
@@ -28,7 +29,8 @@ Pipeline.build
2829
JobSpec::{
2930
dirtyWhen = dirtyWhen,
3031
path = "Test",
31-
name = "RosettaIntegrationTests"
32+
name = "RosettaIntegrationTests",
33+
stage = PipelineStage.Type.Long
3234
}
3335
, steps = [
3436
Command.build

‎buildkite/src/Jobs/Test/RunSnarkProfiler.dhall

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ let S = ../../Lib/SelectFiles.dhall
55
let D = S.PathPattern
66

77
let Pipeline = ../../Pipeline/Dsl.dhall
8+
let PipelineStage = ../../Pipeline/Stage.dhall
89
let JobSpec = ../../Pipeline/JobSpec.dhall
910

1011
let Command = ../../Command/Base.dhall
@@ -54,7 +55,8 @@ Pipeline.build
5455
JobSpec::{
5556
dirtyWhen = lintDirtyWhen,
5657
path = "Test",
57-
name = "RunSnarkProfiler"
58+
name = "RunSnarkProfiler",
59+
stage = PipelineStage.Type.Long
5860
},
5961
steps = [
6062
buildTestCmd Size.Small dependsOn

‎buildkite/src/Jobs/Test/SingleNodeTest.dhall

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ let S = ../../Lib/SelectFiles.dhall
55
let D = S.PathPattern
66

77
let Pipeline = ../../Pipeline/Dsl.dhall
8+
let PipelineStage = ../../Pipeline/Stage.dhall
9+
810
let JobSpec = ../../Pipeline/JobSpec.dhall
911

1012
let Command = ../../Command/Base.dhall
@@ -56,7 +58,8 @@ Pipeline.build
5658
JobSpec::{
5759
dirtyWhen = unitDirtyWhen,
5860
path = "Test",
59-
name = "SingleNodeTest"
61+
name = "SingleNodeTest",
62+
stage = PipelineStage.Type.Long
6063
},
6164
steps = [
6265
buildTestCmd "dev" "src/test/command_line_tests/command_line_tests.exe" Size.XLarge

‎buildkite/src/Jobs/Test/SnarkyJSTest.dhall

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ let S = ../../Lib/SelectFiles.dhall
22
let B = ../../External/Buildkite.dhall
33

44
let Pipeline = ../../Pipeline/Dsl.dhall
5+
let PipelineStage = ../../Pipeline/Stage.dhall
56
let JobSpec = ../../Pipeline/JobSpec.dhall
67

78
let Command = ../../Command/Base.dhall
@@ -24,7 +25,8 @@ Pipeline.build
2425
S.strictlyStart (S.contains "src/lib")
2526
],
2627
path = "Test",
27-
name = "SnarkyJSTest"
28+
name = "SnarkyJSTest",
29+
stage = PipelineStage.Type.Long
2830
},
2931
steps = [
3032
Command.build

‎buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ let S = ../../Lib/SelectFiles.dhall
22

33
let JobSpec = ../../Pipeline/JobSpec.dhall
44
let Pipeline = ../../Pipeline/Dsl.dhall
5+
let PipelineMode = ../../Pipeline/Mode.dhall
6+
let PipelineStage = ../../Pipeline/Stage.dhall
57

68
let TestExecutive = ../../Command/TestExecutive.dhall
79

@@ -29,7 +31,8 @@ in Pipeline.build Pipeline.Config::{
2931
S.strictlyStart (S.contains "automation/terraform/modules/kubernetes/testnet")
3032
],
3133
path = "Test",
32-
name = "TestnetIntegrationTests"
34+
name = "TestnetIntegrationTests",
35+
stage = PipelineStage.Type.Long
3336
},
3437
steps = [
3538
TestExecutive.build "integration_tests",

‎buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ let S = ../../Lib/SelectFiles.dhall
33
let JobSpec = ../../Pipeline/JobSpec.dhall
44
let Pipeline = ../../Pipeline/Dsl.dhall
55
let PipelineMode = ../../Pipeline/Mode.dhall
6+
let PipelineStage = ../../Pipeline/Stage.dhall
67
let TestExecutive = ../../Command/TestExecutive.dhall
78

89
let dependsOn = [
@@ -24,7 +25,8 @@ in Pipeline.build Pipeline.Config::{
2425
],
2526
path = "Test",
2627
name = "TestnetIntegrationTestsLong",
27-
mode = PipelineMode.Type.Stable
28+
mode = PipelineMode.Type.Stable,
29+
stage = PipelineStage.Type.Long
2830
},
2931
steps = [
3032
TestExecutive.execute "hard-fork" dependsOn

‎buildkite/src/Monorepo.dhall

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ let Size = ./Command/Size.dhall
1616
let triggerCommand = ./Pipeline/TriggerCommand.dhall
1717

1818
let mode = env:BUILDKITE_PIPELINE_MODE as Text ? "PullRequest"
19-
let stage = env:BUILDKITE_PIPELINE_STAGE as Text ? "Test"
19+
let stage = env:BUILDKITE_PIPELINE_STAGE as Text ? "Fast"
2020

2121
let jobs : List JobSpec.Type =
2222
List/map

‎buildkite/src/Pipeline/JobSpec.dhall

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,6 @@ in
1717
default = {
1818
path = ".",
1919
mode = PipelineMode.Type.PullRequest,
20-
stage = PipelineStage.Type.Test
20+
stage = PipelineStage.Type.Fast
2121
}
2222
}

0 commit comments

Comments
 (0)