diff --git a/buildkite/src/Command/MinaArtifact.dhall b/buildkite/src/Command/MinaArtifact.dhall index 89ed25e662b..6ef33dfd9ff 100644 --- a/buildkite/src/Command/MinaArtifact.dhall +++ b/buildkite/src/Command/MinaArtifact.dhall @@ -5,6 +5,9 @@ let S = ../Lib/SelectFiles.dhall let D = S.PathPattern let Pipeline = ../Pipeline/Dsl.dhall +let PipelineTag = ../Pipeline/Tag.dhall +let PipelineMode = ../Pipeline/Mode.dhall + let JobSpec = ../Pipeline/JobSpec.dhall let Command = ./Base.dhall @@ -16,13 +19,16 @@ let DebianVersions = ../Constants/DebianVersions.dhall in -let pipeline : DebianVersions.DebVersion -> Pipeline.Config.Type = \(debVersion : DebianVersions.DebVersion) -> +let pipeline : DebianVersions.DebVersion -> PipelineMode.Type -> Pipeline.Config.Type = \(debVersion : DebianVersions.DebVersion) -> +\(mode: PipelineMode.Type) -> Pipeline.Config::{ spec = JobSpec::{ dirtyWhen = DebianVersions.dirtyWhen debVersion, path = "Release", - name = "MinaArtifact${DebianVersions.capitalName debVersion}" + name = "MinaArtifact${DebianVersions.capitalName debVersion}", + tags = [ PipelineTag.Type.Long, PipelineTag.Type.Release ], + mode = mode }, steps = [ Libp2p.step debVersion, @@ -119,7 +125,7 @@ let pipeline : DebianVersions.DebVersion -> Pipeline.Config.Type = \(debVersion in { - bullseye = pipeline DebianVersions.DebVersion.Bullseye - , buster = pipeline DebianVersions.DebVersion.Buster - , focal = pipeline DebianVersions.DebVersion.Focal + bullseye = pipeline DebianVersions.DebVersion.Bullseye PipelineMode.Type.PullRequest + , buster = pipeline DebianVersions.DebVersion.Buster PipelineMode.Type.PullRequest + , focal = pipeline DebianVersions.DebVersion.Focal PipelineMode.Type.PullRequest } diff --git a/buildkite/src/Jobs/Lint/Fast.dhall b/buildkite/src/Jobs/Lint/Fast.dhall index d9645e5b545..fc96486a180 100644 --- a/buildkite/src/Jobs/Lint/Fast.dhall +++ b/buildkite/src/Jobs/Lint/Fast.dhall @@ -5,6 +5,7 @@ let B = ../../External/Buildkite.dhall let S = ../../Lib/SelectFiles.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let JobSpec = ../../Pipeline/JobSpec.dhall @@ -36,6 +37,7 @@ in Pipeline.build ] , path = "Lint" , name = "Fast" + , tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ] } , steps = [ Command.build diff --git a/buildkite/src/Jobs/Lint/HelmChart.dhall b/buildkite/src/Jobs/Lint/HelmChart.dhall index 2b0e602117c..f0f93b132f1 100644 --- a/buildkite/src/Jobs/Lint/HelmChart.dhall +++ b/buildkite/src/Jobs/Lint/HelmChart.dhall @@ -4,6 +4,7 @@ let S = ../../Lib/SelectFiles.dhall let Cmd = ../../Lib/Cmds.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -23,7 +24,8 @@ Pipeline.build S.exactly "buildkite/scripts/helm-ci" "sh" ], path = "Lint", - name = "HelmChart" + name = "HelmChart", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ] }, steps = [ Command.build diff --git a/buildkite/src/Jobs/Lint/Merge.dhall b/buildkite/src/Jobs/Lint/Merge.dhall index f6d06707e46..eae39a7b21a 100644 --- a/buildkite/src/Jobs/Lint/Merge.dhall +++ b/buildkite/src/Jobs/Lint/Merge.dhall @@ -4,6 +4,7 @@ let B = ../../External/Buildkite.dhall let SelectFiles = ../../Lib/SelectFiles.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Cmd = ../../Lib/Cmds.dhall @@ -20,7 +21,8 @@ Pipeline.build spec = JobSpec::{ dirtyWhen = [ SelectFiles.everything ], path = "Lint", - name = "Merge" + name = "Merge", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ] }, steps = [ Command.build diff --git a/buildkite/src/Jobs/Lint/OCaml.dhall b/buildkite/src/Jobs/Lint/OCaml.dhall index a79f1e00026..3dec98290af 100644 --- a/buildkite/src/Jobs/Lint/OCaml.dhall +++ b/buildkite/src/Jobs/Lint/OCaml.dhall @@ -7,6 +7,7 @@ let JobSpec = ../../Pipeline/JobSpec.dhall let Cmd = ../../Lib/Cmds.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let RunInToolchain = ../../Command/RunInToolchain.dhall @@ -31,6 +32,7 @@ in Pipeline.build [ dirtyDhallDir, S.strictlyStart (S.contains "src/") ] , path = "Lint" , name = "OCaml" + , tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ] } , steps = [ Command.build diff --git a/buildkite/src/Jobs/Lint/Rust.dhall b/buildkite/src/Jobs/Lint/Rust.dhall index a551272308d..c099d225646 100644 --- a/buildkite/src/Jobs/Lint/Rust.dhall +++ b/buildkite/src/Jobs/Lint/Rust.dhall @@ -4,6 +4,8 @@ let S = ../../Lib/SelectFiles.dhall let Cmd = ../../Lib/Cmds.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -19,7 +21,8 @@ Pipeline.build spec = JobSpec::{ dirtyWhen = [ S.contains "src/app/trace-tool", S.strictlyStart (S.contains "buildkite/src/Jobs/Lint/Rust") ], path = "Lint", - name = "Rust" + name = "Rust", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ] }, steps = [ Command.build diff --git a/buildkite/src/Jobs/Lint/TestnetAlerts.dhall b/buildkite/src/Jobs/Lint/TestnetAlerts.dhall index 90b69b3f91e..c60772c198b 100644 --- a/buildkite/src/Jobs/Lint/TestnetAlerts.dhall +++ b/buildkite/src/Jobs/Lint/TestnetAlerts.dhall @@ -7,6 +7,8 @@ let S = ../../Lib/SelectFiles.dhall let Cmd = ../../Lib/Cmds.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -25,7 +27,8 @@ Pipeline.build S.strictlyStart (S.contains "buildkite/src/Jobs/Release/TestnetAlerts") ], path = "Lint", - name = "TestnetAlerts" + name = "TestnetAlerts", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ] }, steps = [ Command.build diff --git a/buildkite/src/Jobs/Lint/ValidationService.dhall b/buildkite/src/Jobs/Lint/ValidationService.dhall index 96fe9d19bf8..a2469a29ef0 100644 --- a/buildkite/src/Jobs/Lint/ValidationService.dhall +++ b/buildkite/src/Jobs/Lint/ValidationService.dhall @@ -4,6 +4,8 @@ let List/map = Prelude.List.map let S = ../../Lib/SelectFiles.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall + let Cmd = ../../Lib/Cmds.dhall let Command = ../../Command/Base.dhall let JobSpec = ../../Pipeline/JobSpec.dhall @@ -52,7 +54,8 @@ in Pipeline.build Pipeline.Config::{ S.strictlyStart (S.contains ValidationService.rootPath) ], path = "Lint", - name = "ValidationService" + name = "ValidationService", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ] }, steps = [ Command.build Command.Config::{ diff --git a/buildkite/src/Jobs/Lint/Xrefcheck.dhall b/buildkite/src/Jobs/Lint/Xrefcheck.dhall index 139687b4a5b..28a9268420c 100644 --- a/buildkite/src/Jobs/Lint/Xrefcheck.dhall +++ b/buildkite/src/Jobs/Lint/Xrefcheck.dhall @@ -4,6 +4,8 @@ let B = ../../External/Buildkite.dhall let SelectFiles = ../../Lib/SelectFiles.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Cmd = ../../Lib/Cmds.dhall @@ -23,7 +25,8 @@ Pipeline.build SelectFiles.strictly (SelectFiles.contains ".xrefcheck.yml") ], path = "Lint", - name = "Xrefcheck" + name = "Xrefcheck", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ] }, steps = [ Command.build diff --git a/buildkite/src/Jobs/Release/HelmRelease.dhall b/buildkite/src/Jobs/Release/HelmRelease.dhall index 7fa74079dda..8e843171ab7 100644 --- a/buildkite/src/Jobs/Release/HelmRelease.dhall +++ b/buildkite/src/Jobs/Release/HelmRelease.dhall @@ -4,6 +4,8 @@ let S = ../../Lib/SelectFiles.dhall let Cmd = ../../Lib/Cmds.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -21,7 +23,8 @@ Pipeline.build S.exactly "buildkite/scripts/helm-ci" "sh" ], path = "Release", - name = "HelmRelease" + name = "HelmRelease", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Release ] }, steps = [ Command.build diff --git a/buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall b/buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall index e1232b6278e..99e193cf299 100644 --- a/buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall +++ b/buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall @@ -5,6 +5,8 @@ let S = ../../Lib/SelectFiles.dhall let D = S.PathPattern let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -29,7 +31,8 @@ Pipeline.build S.strictlyStart (S.contains "src/app/itn_orchestrator") ], path = "Release", - name = "ItnOrchestratorArtifact" + name = "ItnOrchestratorArtifact", + tags = [ PipelineTag.Type.Long, PipelineTag.Type.Release ] }, steps = [ DockerImage.generateStep spec diff --git a/buildkite/src/Jobs/Release/LeaderboardArtifact.dhall b/buildkite/src/Jobs/Release/LeaderboardArtifact.dhall index f8b657359fd..252cd012e22 100644 --- a/buildkite/src/Jobs/Release/LeaderboardArtifact.dhall +++ b/buildkite/src/Jobs/Release/LeaderboardArtifact.dhall @@ -5,6 +5,8 @@ let S = ../../Lib/SelectFiles.dhall let D = S.PathPattern let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -27,7 +29,8 @@ Pipeline.build S.strictlyStart (S.contains "frontend/leaderboard") ], path = "Release", - name = "LeaderboardArtifact" + name = "LeaderboardArtifact", + tags = [ PipelineTag.Type.Long, PipelineTag.Type.Release ] }, steps = [ Command.build diff --git a/buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall b/buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall new file mode 100644 index 00000000000..0e2b4f198ea --- /dev/null +++ b/buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall @@ -0,0 +1,50 @@ +let Prelude = ../../External/Prelude.dhall + +let Cmd = ../../Lib/Cmds.dhall +let S = ../../Lib/SelectFiles.dhall + +let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall +let JobSpec = ../../Pipeline/JobSpec.dhall + +let Command = ../../Command/Base.dhall +let Size = ../../Command/Size.dhall +let DockerImage = ../../Command/DockerImage.dhall +let DockerLogin = ../../Command/DockerLogin/Type.dhall + + +in + +Pipeline.build + Pipeline.Config::{ + spec = + JobSpec::{ + dirtyWhen = [ + S.strictlyStart (S.contains "dockerfiles/stages/1-"), + S.strictlyStart (S.contains "dockerfiles/stages/2-"), + S.strictlyStart (S.contains "dockerfiles/stages/3-"), + S.strictlyStart (S.contains "buildkite/src/Jobs/Release/MinaToolchainArtifact"), + S.strictly (S.contains "opam.export"), + -- Rust version has changed + S.strictlyEnd (S.contains "rust-toolchain.toml") + ], + path = "Release", + name = "MinaToolchainArtifactBullseye", + tags = [ PipelineTag.Type.Long, PipelineTag.Type.Release ] + }, + steps = [ + + -- mina-toolchain Debian 11 "Bullseye" Toolchain + let toolchainBullseyeSpec = DockerImage.ReleaseSpec::{ + service="mina-toolchain", + deb_codename="bullseye", + extra_args="--no-cache", + step_key="toolchain-bullseye-docker-image" + } + + in + + DockerImage.generateStep toolchainBullseyeSpec + + ] + } \ No newline at end of file diff --git a/buildkite/src/Jobs/Release/MinaToolchainArtifact.dhall b/buildkite/src/Jobs/Release/MinaToolchainArtifactBuster.dhall similarity index 77% rename from buildkite/src/Jobs/Release/MinaToolchainArtifact.dhall rename to buildkite/src/Jobs/Release/MinaToolchainArtifactBuster.dhall index 3780943bb12..f478a107890 100644 --- a/buildkite/src/Jobs/Release/MinaToolchainArtifact.dhall +++ b/buildkite/src/Jobs/Release/MinaToolchainArtifactBuster.dhall @@ -4,6 +4,8 @@ let Cmd = ../../Lib/Cmds.dhall let S = ../../Lib/SelectFiles.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall +let PipelineMode = ../../Pipeline/Mode.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -28,22 +30,12 @@ Pipeline.build S.strictlyEnd (S.contains "rust-toolchain.toml") ], path = "Release", - name = "MinaToolchainArtifact" + name = "MinaToolchainArtifactBuster", + tags = [ PipelineTag.Type.Long, PipelineTag.Type.Release ], + mode = PipelineMode.Type.Stable }, steps = [ - -- mina-toolchain Debian 11 "Bullseye" Toolchain - let toolchainBullseyeSpec = DockerImage.ReleaseSpec::{ - service="mina-toolchain", - deb_codename="bullseye", - extra_args="--no-cache", - step_key="toolchain-bullseye-docker-image" - } - - in - - DockerImage.generateStep toolchainBullseyeSpec, - -- mina-toolchain Debian 10 "Buster" Toolchain let toolchainBusterSpec = DockerImage.ReleaseSpec::{ service="mina-toolchain", diff --git a/buildkite/src/Jobs/Release/TestnetAlerts.dhall b/buildkite/src/Jobs/Release/TestnetAlerts.dhall index 06125455a8c..e444c4e7a33 100644 --- a/buildkite/src/Jobs/Release/TestnetAlerts.dhall +++ b/buildkite/src/Jobs/Release/TestnetAlerts.dhall @@ -7,6 +7,8 @@ let S = ../../Lib/SelectFiles.dhall let Cmd = ../../Lib/Cmds.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -24,7 +26,8 @@ Pipeline.build S.strictlyStart (S.contains "buildkite/src/Jobs/Release/TestnetAlerts") ], path = "Release", - name = "TestnetAlerts" + name = "TestnetAlerts", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Release ] }, steps = [ Command.build diff --git a/buildkite/src/Jobs/Release/TraceTool.dhall b/buildkite/src/Jobs/Release/TraceTool.dhall index d4a0e4997f4..0d6770523e5 100644 --- a/buildkite/src/Jobs/Release/TraceTool.dhall +++ b/buildkite/src/Jobs/Release/TraceTool.dhall @@ -4,6 +4,8 @@ let S = ../../Lib/SelectFiles.dhall let Cmd = ../../Lib/Cmds.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -20,7 +22,8 @@ Pipeline.build spec = JobSpec::{ dirtyWhen = [ S.contains "src/app/trace-tool", S.strictlyStart (S.contains "buildkite/src/Jobs/TraceTool") ], path = "Release", - name = "TraceTool" + name = "TraceTool", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Release ] }, steps = [ Command.build diff --git a/buildkite/src/Jobs/Test/ArchiveNodeUnitTest.dhall b/buildkite/src/Jobs/Test/ArchiveNodeUnitTest.dhall index e92cf6be77b..16fa19b1cc1 100644 --- a/buildkite/src/Jobs/Test/ArchiveNodeUnitTest.dhall +++ b/buildkite/src/Jobs/Test/ArchiveNodeUnitTest.dhall @@ -2,6 +2,7 @@ let Prelude = ../../External/Prelude.dhall let Cmd = ../../Lib/Cmds.dhall let S = ../../Lib/SelectFiles.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall let RunInToolchain = ../../Command/RunInToolchain.dhall @@ -27,6 +28,7 @@ Pipeline.build ] , path = "Test" , name = "ArchiveNodeUnitTest" + , tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ] } , steps = let outerDir : Text = diff --git a/buildkite/src/Jobs/Test/CheckDhall.dhall b/buildkite/src/Jobs/Test/CheckDhall.dhall index 34f8c78476e..18cace107aa 100644 --- a/buildkite/src/Jobs/Test/CheckDhall.dhall +++ b/buildkite/src/Jobs/Test/CheckDhall.dhall @@ -3,6 +3,7 @@ let D = S.PathPattern let JobSpec = ../../Pipeline/JobSpec.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let Command = ../../Command/Base.dhall let Docker = ../../Command/Docker/Type.dhall let Size = ../../Command/Size.dhall @@ -23,7 +24,8 @@ Pipeline.build S.exactly "buildkite/scripts/generate-jobs" "sh" ], path = "Test", - name = "CheckDhall" + name = "CheckDhall", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ] }, steps = [ Command.build diff --git a/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall b/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall index be248e22210..4583a5b38bc 100644 --- a/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall +++ b/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall @@ -2,6 +2,7 @@ let S = ../../Lib/SelectFiles.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let CheckGraphQLSchema = ../../Command/CheckGraphQLSchema.dhall @@ -18,7 +19,8 @@ in Pipeline.build Pipeline.Config::{ S.strictly (S.contains "Makefile") ], path = "Test", - name = "CheckGraphQLSchema" + name = "CheckGraphQLSchema", + tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] }, steps = [ CheckGraphQLSchema.step dependsOn diff --git a/buildkite/src/Jobs/Test/ConnectToBerkeley.dhall b/buildkite/src/Jobs/Test/ConnectToBerkeley.dhall index d6f404bfcf5..4537de014e9 100644 --- a/buildkite/src/Jobs/Test/ConnectToBerkeley.dhall +++ b/buildkite/src/Jobs/Test/ConnectToBerkeley.dhall @@ -2,6 +2,7 @@ let S = ../../Lib/SelectFiles.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let ConnectToTestnet = ../../Command/ConnectToTestnet.dhall @@ -19,7 +20,8 @@ in Pipeline.build Pipeline.Config::{ S.exactly "buildkite/src/Command/ConnectToTestnet" "dhall" ], path = "Test", - name = "ConnectToBerkeley" + name = "ConnectToBerkeley", + tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] }, steps = [ ConnectToTestnet.step dependsOn diff --git a/buildkite/src/Jobs/Test/CoverageTearDown.dhall b/buildkite/src/Jobs/Test/CoverageTearDown.dhall index bba020d8025..aa10f2a5ff8 100644 --- a/buildkite/src/Jobs/Test/CoverageTearDown.dhall +++ b/buildkite/src/Jobs/Test/CoverageTearDown.dhall @@ -8,7 +8,7 @@ let Size = ../../Command/Size.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Pipeline = ../../Pipeline/Dsl.dhall let PipelineMode = ../../Pipeline/Mode.dhall -let PipelineStage = ../../Pipeline/Stage.dhall +let PipelineTag = ../../Pipeline/Tag.dhall in Pipeline.build Pipeline.Config::{ @@ -16,7 +16,7 @@ in Pipeline.build JobSpec::{ dirtyWhen = [ S.everything ], path = "Test", - stage = PipelineStage.Type.TearDown, + tags = [ PipelineTag.Type.TearDown ], name = "CoverageTearDown" } , steps = [ diff --git a/buildkite/src/Jobs/Test/DaemonUnitTest.dhall b/buildkite/src/Jobs/Test/DaemonUnitTest.dhall index 7b769901aa7..a21bc935750 100644 --- a/buildkite/src/Jobs/Test/DaemonUnitTest.dhall +++ b/buildkite/src/Jobs/Test/DaemonUnitTest.dhall @@ -5,6 +5,8 @@ let S = ../../Lib/SelectFiles.dhall let D = S.PathPattern let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -44,7 +46,8 @@ Pipeline.build JobSpec::{ dirtyWhen = unitDirtyWhen, path = "Test", - name = "DaemonUnitTest" + name = "DaemonUnitTest", + tags = [ PipelineTag.Type.VeryLong, PipelineTag.Type.Test ] }, steps = [ buildTestCmd "dev" "src/lib" Size.XLarge diff --git a/buildkite/src/Jobs/Test/DelegationBackendUnitTest.dhall b/buildkite/src/Jobs/Test/DelegationBackendUnitTest.dhall index f0b8e42f78e..a41a66fce87 100644 --- a/buildkite/src/Jobs/Test/DelegationBackendUnitTest.dhall +++ b/buildkite/src/Jobs/Test/DelegationBackendUnitTest.dhall @@ -4,6 +4,8 @@ let Cmd = ../../Lib/Cmds.dhall let S = ../../Lib/SelectFiles.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -22,7 +24,8 @@ Pipeline.build S.exactly "buildkite/src/Jobs/Test/DelegationBackendUnitTest" "dhall" ], path = "Test", - name = "DelegationBackendUnitTest" + name = "DelegationBackendUnitTest", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ] }, steps = [ Command.build diff --git a/buildkite/src/Jobs/Test/FuzzyZkappTest.dhall b/buildkite/src/Jobs/Test/FuzzyZkappTest.dhall index 62ea69f2724..0d3ccc55cc4 100644 --- a/buildkite/src/Jobs/Test/FuzzyZkappTest.dhall +++ b/buildkite/src/Jobs/Test/FuzzyZkappTest.dhall @@ -6,8 +6,9 @@ let D = S.PathPattern let Pipeline = ../../Pipeline/Dsl.dhall let PipelineMode = ../../Pipeline/Mode.dhall -let JobSpec = ../../Pipeline/JobSpec.dhall +let PipelineTag = ../../Pipeline/Tag.dhall +let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall let RunInToolchain = ../../Command/RunInToolchain.dhall let Docker = ../../Command/Docker/Type.dhall @@ -46,6 +47,7 @@ Pipeline.build dirtyWhen = unitDirtyWhen, path = "Test", name = "FuzzyZkappTest", + tags = [ PipelineTag.Type.VeryLong, PipelineTag.Type.Test ], mode = PipelineMode.Type.Stable }, steps = [ diff --git a/buildkite/src/Jobs/Test/Libp2pUnitTest.dhall b/buildkite/src/Jobs/Test/Libp2pUnitTest.dhall index e22384dc2ba..76098a7f444 100644 --- a/buildkite/src/Jobs/Test/Libp2pUnitTest.dhall +++ b/buildkite/src/Jobs/Test/Libp2pUnitTest.dhall @@ -4,6 +4,7 @@ let Cmd = ../../Lib/Cmds.dhall let S = ../../Lib/SelectFiles.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -23,7 +24,8 @@ Pipeline.build S.exactly "buildkite/src/Jobs/Test/Libp2pUnitTest" "dhall" ], path = "Test", - name = "Libp2pUnitTest" + name = "Libp2pUnitTest", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ] }, steps = [ Command.build diff --git a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall index 4b8d0c73302..06f2b20473c 100644 --- a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall +++ b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall @@ -5,6 +5,7 @@ let Cmd = ../../Lib/Cmds.dhall let S = ../../Lib/SelectFiles.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -32,7 +33,8 @@ Pipeline.build JobSpec::{ dirtyWhen = dirtyWhen, path = "Test", - name = "RosettaIntegrationTests" + name = "RosettaIntegrationTests", + tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] } , steps = [ Command.build diff --git a/buildkite/src/Jobs/Test/RosettaUnitTest.dhall b/buildkite/src/Jobs/Test/RosettaUnitTest.dhall index 7e802917eff..6375974d206 100644 --- a/buildkite/src/Jobs/Test/RosettaUnitTest.dhall +++ b/buildkite/src/Jobs/Test/RosettaUnitTest.dhall @@ -5,6 +5,7 @@ let S = ../../Lib/SelectFiles.dhall let D = S.PathPattern let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -41,7 +42,8 @@ Pipeline.build JobSpec::{ dirtyWhen = unitDirtyWhen, path = "Test", - name = "RosettaUnitTest" + name = "RosettaUnitTest", + tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] }, steps = [ buildTestCmd "dev" "src/app/rosetta" Size.Small diff --git a/buildkite/src/Jobs/Test/RunSnarkProfiler.dhall b/buildkite/src/Jobs/Test/RunSnarkProfiler.dhall index dee246854be..7a8b0251f5a 100644 --- a/buildkite/src/Jobs/Test/RunSnarkProfiler.dhall +++ b/buildkite/src/Jobs/Test/RunSnarkProfiler.dhall @@ -5,6 +5,7 @@ let S = ../../Lib/SelectFiles.dhall let D = S.PathPattern let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -54,7 +55,8 @@ Pipeline.build JobSpec::{ dirtyWhen = lintDirtyWhen, path = "Test", - name = "RunSnarkProfiler" + name = "RunSnarkProfiler", + tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] }, steps = [ buildTestCmd Size.Small dependsOn diff --git a/buildkite/src/Jobs/Test/SingleNodeTest.dhall b/buildkite/src/Jobs/Test/SingleNodeTest.dhall index 06ed548a2f3..8f85ec04140 100644 --- a/buildkite/src/Jobs/Test/SingleNodeTest.dhall +++ b/buildkite/src/Jobs/Test/SingleNodeTest.dhall @@ -5,6 +5,8 @@ let S = ../../Lib/SelectFiles.dhall let D = S.PathPattern let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -56,7 +58,8 @@ Pipeline.build JobSpec::{ dirtyWhen = unitDirtyWhen, path = "Test", - name = "SingleNodeTest" + name = "SingleNodeTest", + tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] }, steps = [ buildTestCmd "dev" "src/test/command_line_tests/command_line_tests.exe" Size.XLarge diff --git a/buildkite/src/Jobs/Test/SnarkyJSTest.dhall b/buildkite/src/Jobs/Test/SnarkyJSTest.dhall index d5054981a12..b2027f9863b 100644 --- a/buildkite/src/Jobs/Test/SnarkyJSTest.dhall +++ b/buildkite/src/Jobs/Test/SnarkyJSTest.dhall @@ -2,6 +2,7 @@ let S = ../../Lib/SelectFiles.dhall let B = ../../External/Buildkite.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -24,7 +25,8 @@ Pipeline.build S.strictlyStart (S.contains "src/lib") ], path = "Test", - name = "SnarkyJSTest" + name = "SnarkyJSTest", + tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] }, steps = [ Command.build diff --git a/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall b/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall index 96f6ab5e835..2e918ff3432 100644 --- a/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall +++ b/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall @@ -3,6 +3,8 @@ let S = ../../Lib/SelectFiles.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Pipeline = ../../Pipeline/Dsl.dhall let PipelineMode = ../../Pipeline/Mode.dhall +let PipelineTag = ../../Pipeline/Tag.dhall + let TestExecutive = ../../Command/TestExecutive.dhall let dependsOn = [ @@ -28,6 +30,7 @@ in Pipeline.build Pipeline.Config::{ ], path = "Test", name = "TestnetIntegrationTests", + tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ], mode = PipelineMode.Type.Stable }, steps = [ diff --git a/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall b/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall index 04dc26cef28..271a99d08e9 100644 --- a/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall +++ b/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall @@ -3,6 +3,7 @@ let S = ../../Lib/SelectFiles.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Pipeline = ../../Pipeline/Dsl.dhall let PipelineMode = ../../Pipeline/Mode.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let TestExecutive = ../../Command/TestExecutive.dhall let dependsOn = [ @@ -23,7 +24,8 @@ in Pipeline.build Pipeline.Config::{ ], path = "Test", name = "TestnetIntegrationTestsLong", - mode = PipelineMode.Type.Stable + mode = PipelineMode.Type.Stable, + tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] }, steps = [ TestExecutive.execute "hard-fork" dependsOn diff --git a/buildkite/src/Jobs/Test/ValidationService.dhall b/buildkite/src/Jobs/Test/ValidationService.dhall index 1c2ee6fa571..cce41331433 100644 --- a/buildkite/src/Jobs/Test/ValidationService.dhall +++ b/buildkite/src/Jobs/Test/ValidationService.dhall @@ -1,6 +1,7 @@ let S = ../../Lib/SelectFiles.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let Command = ../../Command/Base.dhall let Docker = ../../Command/Docker/Type.dhall let Size = ../../Command/Size.dhall @@ -15,7 +16,8 @@ in Pipeline.build Pipeline.Config::{ S.strictlyStart (S.contains ValidationService.rootPath) ], path = "Test", - name = "ValidationService" + name = "ValidationService", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ] }, steps = [ Command.build Command.Config::{ diff --git a/buildkite/src/Jobs/Test/VersionLint.dhall b/buildkite/src/Jobs/Test/VersionLint.dhall index fcbdc34f21a..a0aa132649a 100644 --- a/buildkite/src/Jobs/Test/VersionLint.dhall +++ b/buildkite/src/Jobs/Test/VersionLint.dhall @@ -5,6 +5,7 @@ let S = ../../Lib/SelectFiles.dhall let D = S.PathPattern let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -56,7 +57,8 @@ Pipeline.build JobSpec::{ dirtyWhen = lintDirtyWhen, path = "Test", - name = "VersionLint" + name = "VersionLint", + tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] }, steps = [ buildTestCmd "develop" Size.Small dependsOn diff --git a/buildkite/src/Jobs/Test/ZkappMetrics.dhall b/buildkite/src/Jobs/Test/ZkappMetrics.dhall index 4bf912a70c4..31ec5072d7b 100644 --- a/buildkite/src/Jobs/Test/ZkappMetrics.dhall +++ b/buildkite/src/Jobs/Test/ZkappMetrics.dhall @@ -2,6 +2,7 @@ let Prelude = ../../External/Prelude.dhall let S = ../../Lib/SelectFiles.dhall let Cmd = ../../Lib/Cmds.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let Command = ../../Command/Base.dhall let RunInToolchain = ../../Command/RunInToolchain.dhall let WithCargo = ../../Command/WithCargo.dhall @@ -20,6 +21,7 @@ Pipeline.build ] , path = "Test" , name = "ZkappMetrics" + , tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ] } , steps = [ Command.build diff --git a/buildkite/src/Jobs/Test/ZkappTestToolUnitTest.dhall b/buildkite/src/Jobs/Test/ZkappTestToolUnitTest.dhall index 45b00d8c2b5..7d0996abe30 100644 --- a/buildkite/src/Jobs/Test/ZkappTestToolUnitTest.dhall +++ b/buildkite/src/Jobs/Test/ZkappTestToolUnitTest.dhall @@ -5,6 +5,7 @@ let S = ../../Lib/SelectFiles.dhall let D = S.PathPattern let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -41,7 +42,8 @@ Pipeline.build JobSpec::{ dirtyWhen = unitDirtyWhen, path = "Test", - name = "ZkappTestToolUnitTest" + name = "ZkappTestToolUnitTest", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ] }, steps = [ buildTestCmd "dev" "src/app/zkapp_test_transaction" Size.Small diff --git a/buildkite/src/Jobs/Test/ZkappsExamplesTest.dhall b/buildkite/src/Jobs/Test/ZkappsExamplesTest.dhall index 0fcdc592d98..9a6e4be6c95 100644 --- a/buildkite/src/Jobs/Test/ZkappsExamplesTest.dhall +++ b/buildkite/src/Jobs/Test/ZkappsExamplesTest.dhall @@ -5,6 +5,7 @@ let S = ../../Lib/SelectFiles.dhall let D = S.PathPattern let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -43,7 +44,8 @@ Pipeline.build JobSpec::{ dirtyWhen = unitDirtyWhen, path = "Test", - name = "ZkappsExamplesTest" + name = "ZkappsExamplesTest", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ] }, steps = [ buildTestCmd "dev" Size.XLarge diff --git a/buildkite/src/Monorepo.dhall b/buildkite/src/Monorepo.dhall index 64783a0582f..3281333b8e6 100644 --- a/buildkite/src/Monorepo.dhall +++ b/buildkite/src/Monorepo.dhall @@ -11,12 +11,14 @@ let Docker = ./Command/Docker/Type.dhall let JobSpec = ./Pipeline/JobSpec.dhall let Pipeline = ./Pipeline/Dsl.dhall let PipelineMode = ./Pipeline/Mode.dhall -let PipelineStage = ./Pipeline/Stage.dhall +let PipelineFilter = ./Pipeline/Filter.dhall +let PipelineTag = ./Pipeline/Tag.dhall let Size = ./Command/Size.dhall let triggerCommand = ./Pipeline/TriggerCommand.dhall let mode = env:BUILDKITE_PIPELINE_MODE as Text ? "PullRequest" -let stage = env:BUILDKITE_PIPELINE_STAGE as Text ? "Test" +let include_tags = env:BUILDKITE_PIPELINE_TAGS_INCL as Text ? "Fast" +let exclude_tags = env:BUILDKITE_PIPELINE_TAGS_EXCL as Text ? "" let jobs : List JobSpec.Type = List/map @@ -33,34 +35,36 @@ let prefixCommands = [ -- Run a job if we touched a dirty path -let commands: Text -> Text -> List Cmd.Type = \(targetStage: Text) -> \(targetMode: Text) -> +let commands: PipelineFilter.Type -> PipelineMode.Type -> List Cmd.Type = \(filter: PipelineFilter.Type) -> \(mode: PipelineMode.Type) -> Prelude.List.map JobSpec.Type Cmd.Type (\(job: JobSpec.Type) -> let jobMode = PipelineMode.capitalName job.mode - let jobStage = PipelineStage.capitalName job.stage - + let targetMode = PipelineMode.capitalName mode + let targetTags = PipelineFilter.tags filter + let filter = PipelineFilter.show filter + let isIncluded = Prelude.Bool.show (PipelineTag.contains job.tags targetTags) let dirtyWhen = SelectFiles.compile job.dirtyWhen let trigger = triggerCommand "src/Jobs/${job.path}/${job.name}.dhall" let pipelineHandlers = { PullRequest = '' if [ "${targetMode}" == "PullRequest" ]; then - if [ "${jobStage}" == "${targetStage}" ]; then + if [ "${isIncluded}" == "True" ]; then if (cat _computed_diff.txt | egrep -q '${dirtyWhen}'); then echo "Triggering ${job.name} for reason:" cat _computed_diff.txt | egrep '${dirtyWhen}' ${Cmd.format trigger} fi else - echo "Skipping ${job.name} because this is a ${targetStage} stage" + echo "Skipping ${job.name} because this is a ${filter} stage" fi else - if [ "${jobStage}" == "${targetStage}" ]; then + if [ "${isIncluded}" == "True" ]; then echo "Triggering ${job.name} because this is a stable buildkite run" ${Cmd.format trigger} else - echo "Skipping ${job.name} because this is a ${targetStage} stage" + echo "Skipping ${job.name} because this is a ${filter} stage" fi fi '', @@ -68,11 +72,11 @@ let commands: Text -> Text -> List Cmd.Type = \(targetStage: Text) -> \(target if [ "${targetMode}" == "PullRequest" ]; then echo "Skipping ${job.name} because this is a PR buildkite run" else - if [ "${jobStage}" == "${targetStage}" ]; then + if [ "${isIncluded}" == "True" ]; then echo "Triggering ${job.name} because this is a stable buildkite run" ${Cmd.format trigger} else - echo "Skipping ${job.name} because this is a ${targetStage} stage" + echo "Skipping ${job.name} because this is a ${filter} stage" fi fi '' @@ -81,23 +85,29 @@ let commands: Text -> Text -> List Cmd.Type = \(targetStage: Text) -> \(target ) jobs -in Pipeline.build Pipeline.Config::{ - spec = JobSpec::{ - name = "monorepo-triage", - -- TODO: Clean up this code so we don't need an unused dirtyWhen here - dirtyWhen = [ SelectFiles.everything ] - }, - steps = [ - Command.build - Command.Config::{ - commands = prefixCommands # (commands stage mode), - label = "Monorepo triage ${stage}", - key = "cmds-${stage}", - target = Size.Small, - docker = Some Docker::{ - image = (./Constants/ContainerImages.dhall).toolchainBase, - environment = ["BUILDKITE_AGENT_ACCESS_TOKEN", "BUILDKITE_INCREMENTAL"] - } - } - ] -} +in + +(\(args : { filter : PipelineFilter.Type, mode: PipelineMode.Type }) -> + let pipelineType = Pipeline.build Pipeline.Config::{ + spec = JobSpec::{ + name = "monorepo-triage-${PipelineFilter.show args.filter}", + -- TODO: Clean up this code so we don't need an unused dirtyWhen here + dirtyWhen = [ SelectFiles.everything ] + }, + steps = [ + Command.build + Command.Config::{ + commands = prefixCommands # (commands args.filter args.mode), + label = "Monorepo triage ${PipelineFilter.show args.filter}", + key = "cmds-${PipelineFilter.show args.filter}", + target = Size.Small, + docker = Some Docker::{ + image = (./Constants/ContainerImages.dhall).toolchainBase, + environment = ["BUILDKITE_AGENT_ACCESS_TOKEN", "BUILDKITE_INCREMENTAL"] + } + } + ] + } + in pipelineType.pipeline +) + diff --git a/buildkite/src/Pipeline/Filter.dhall b/buildkite/src/Pipeline/Filter.dhall new file mode 100644 index 00000000000..fc9cd7dd5ac --- /dev/null +++ b/buildkite/src/Pipeline/Filter.dhall @@ -0,0 +1,33 @@ +-- Tag defines pipeline +-- Using tags one can tailor pipeline for any need. Each job should be tagged with one or several tags +-- then on pipeline settings we can define which tagged jobs to include or exclue in pipeline + +let Prelude = ../External/Prelude.dhall +let Tag = ./Tag.dhall + +let Filter : Type = < FastOnly | Long | LongAndVeryLong | TearDownOnly | AllTests > + +let tags: Filter -> List Tag.Type = \(filter: Filter) -> + merge { + FastOnly = [ Tag.Type.Fast ] + , LongAndVeryLong = [ Tag.Type.Long, Tag.Type.VeryLong ] + , Long = [ Tag.Type.Long ] + , TearDownOnly = [ Tag.Type.TearDown ] + , AllTests = [ Tag.Type.Lint, Tag.Type.Release, Tag.Type.Test ] + } filter + +let show: Filter -> Text = \(filter: Filter) -> + merge { + FastOnly = "FastOnly" + , LongAndVeryLong = "LongAndVeryLong" + , Long = "Long" + , TearDownOnly = "TearDownOnly" + , AllTests = "AllTests" + } filter + +in +{ + Type = Filter, + tags = tags, + show = show +} \ No newline at end of file diff --git a/buildkite/src/Pipeline/JobSpec.dhall b/buildkite/src/Pipeline/JobSpec.dhall index 187835c7d90..ea0373a0c2d 100644 --- a/buildkite/src/Pipeline/JobSpec.dhall +++ b/buildkite/src/Pipeline/JobSpec.dhall @@ -1,6 +1,6 @@ let SelectFiles = ../Lib/SelectFiles.dhall let PipelineMode = ./Mode.dhall -let PipelineStage = ./Stage.dhall +let PipelineTag = ./Tag.dhall in @@ -11,12 +11,12 @@ in path: Text, name: Text, mode: PipelineMode.Type, - stage: PipelineStage.Type, + tags: List PipelineTag.Type, dirtyWhen: List SelectFiles.Type }, default = { path = ".", mode = PipelineMode.Type.PullRequest, - stage = PipelineStage.Type.Test + tags = [ PipelineTag.Type.Fast ] } } diff --git a/buildkite/src/Pipeline/Stage.dhall b/buildkite/src/Pipeline/Stage.dhall deleted file mode 100644 index fffad54c7a2..00000000000 --- a/buildkite/src/Pipeline/Stage.dhall +++ /dev/null @@ -1,44 +0,0 @@ --- Mode defines pipeline stages --- --- A pipeline in order to be faster and more cost efficient can have up to 3 stages --- Between each stages there is a '- wait' step defined which cause buildkite to wait --- for ALL jobs to complete before running any job from next stage. --- Current design defines three stages: --- - Stage 1 -> contains fastest and most independent jobs which are supposed to provide quickest feedback possible --- - Stage 2 -> contains heavy jobs that should be run only on clean code (no merges issues or lints problems) --- - Tear down -> should contains all clean up or reporting jobs. For example test coverage gathering - -let Prelude = ../External/Prelude.dhall - -let Stage : Type = < Test | TearDown > - -let toNatural: Stage -> Natural = \(stage: Stage) -> - merge { - Test = 1 - , TearDown = 2 - } stage - -let equal: Stage -> Stage -> Bool = \(left: Stage) -> \(right: Stage) -> - Prelude.Natural.equal (toNatural left) (toNatural right) - -let capitalName = \(stage : Stage) -> - merge { - Test = "Test" - , TearDown = "TearDown" - } stage - -let lowerName = \(stage : Stage) -> - merge { - Test = "test" - , TearDown = "tearDown" - } stage - - -in -{ - Type = Stage, - capitalName = capitalName, - lowerName = lowerName, - toNatural = toNatural, - equal = equal -} \ No newline at end of file diff --git a/buildkite/src/Pipeline/Tag.dhall b/buildkite/src/Pipeline/Tag.dhall new file mode 100644 index 00000000000..b6018062e67 --- /dev/null +++ b/buildkite/src/Pipeline/Tag.dhall @@ -0,0 +1,63 @@ +-- Tag defines pipeline +-- Using tags one can tailor pipeline for any need. Each job should be tagged with one or several tags +-- then on pipeline settings we can define which tagged jobs to include or exclue in pipeline + +let Prelude = ../External/Prelude.dhall +let List/any = Prelude.List.any + +let Tag : Type = < Fast | Long | VeryLong | TearDown | Lint | Release | Test > + +let toNatural: Tag -> Natural = \(tag: Tag) -> + merge { + Fast = 1 + , Long = 2 + , VeryLong = 3 + , TearDown = 4 + , Lint = 5 + , Release = 6 + , Test = 7 + } tag + +let equal: Tag -> Tag -> Bool = \(left: Tag) -> \(right: Tag) -> + Prelude.Natural.equal (toNatural left) (toNatural right) + + +let hasAny: Tag -> List Tag -> Bool = \(input: Tag) -> \(tags: List Tag) -> + List/any Tag (\(x: Tag) -> equal x input ) tags + +let contains: List Tag -> List Tag -> Bool = \(input: List Tag) -> \(tags: List Tag) -> + List/any Tag (\(x: Tag) -> hasAny x tags ) input + +let capitalName = \(tag : Tag) -> + merge { + Fast = "Fast" + , Long = "Long" + , VeryLong = "VeryLong" + , TearDown = "TearDown" + , Lint = "Lint" + , Release = "Release" + , Test = "Test" + } tag + +let lowerName = \(tag : Tag) -> + merge { + Fast = "fast" + , Long = "long" + , VeryLong = "veryLong" + , TearDown = "tearDown" + , Lint = "lint" + , Release = "release" + , Test = "test" + } tag + + +in +{ + Type = Tag, + capitalName = capitalName, + lowerName = lowerName, + toNatural = toNatural, + equal = equal, + hasAny = hasAny, + contains = contains +} \ No newline at end of file diff --git a/buildkite/src/Prepare.dhall b/buildkite/src/Prepare.dhall index 101e22be770..5b78ff24369 100644 --- a/buildkite/src/Prepare.dhall +++ b/buildkite/src/Prepare.dhall @@ -8,11 +8,14 @@ let Command = ./Command/Base.dhall let Docker = ./Command/Docker/Type.dhall let JobSpec = ./Pipeline/JobSpec.dhall let Pipeline = ./Pipeline/Dsl.dhall +let PipelineMode = ./Pipeline/Mode.dhall +let PipelineFilter = ./Pipeline/Filter.dhall +let PipelineTag = ./Pipeline/Tag.dhall let Size = ./Command/Size.dhall let triggerCommand = ./Pipeline/TriggerCommand.dhall let mode = env:BUILDKITE_PIPELINE_MODE as Text ? "PullRequest" -let stage = env:BUILDKITE_PIPELINE_STAGE as Text ? "Test" +let filter = env:BUILDKITE_PIPELINE_FILTER as Text ? "FastOnly" let config : Pipeline.Config.Type = Pipeline.Config::{ spec = JobSpec::{ @@ -24,12 +27,12 @@ let config : Pipeline.Config.Type = Pipeline.Config::{ Command.build Command.Config::{ commands = [ Cmd.run "export BUILDKITE_PIPELINE_MODE=${mode}", - Cmd.run "export BUILDKITE_PIPELINE_STAGE=${stage}", + Cmd.run "export BUILDKITE_PIPELINE_FILTER=${filter}", Cmd.run "./buildkite/scripts/generate-jobs.sh > buildkite/src/gen/Jobs.dhall", - triggerCommand "src/Monorepo.dhall" + Cmd.quietly "dhall-to-yaml --quoted <<< '(./buildkite/src/Monorepo.dhall) { mode=(./buildkite/src/Pipeline/Mode.dhall).Type.${mode}, filter=(./buildkite/src/Pipeline/Filter.dhall).Type.${filter} }' | buildkite-agent pipeline upload" ], label = "Prepare monorepo triage", - key = "monorepo-${stage}", + key = "monorepo-${mode}-${filter}", target = Size.Small, docker = Some Docker::{ image = (./Constants/ContainerImages.dhall).toolchainBase,