From 126ab66cf47af51f2840c522d8eb736b4011cb65 Mon Sep 17 00:00:00 2001 From: dkijania Date: Wed, 4 Sep 2024 21:13:51 +0200 Subject: [PATCH] Remove berkeley codename and fix step dependencies --- buildkite/src/Command/DockerImage.dhall | 57 ++++-- .../Command/HardforkPackageGeneration.dhall | 52 ++--- buildkite/src/Command/MinaArtifact.dhall | 179 ++++++++---------- .../src/Command/Promotion/PromoteDebian.dhall | 2 +- .../src/Command/Promotion/PromoteDocker.dhall | 2 +- buildkite/src/Command/ReplayerTest.dhall | 2 +- buildkite/src/Command/RunWithPostgres.dhall | 24 +-- buildkite/src/Constants/Artifacts.dhall | 47 +++-- buildkite/src/Constants/DebianVersions.dhall | 46 ++--- buildkite/src/Constants/DockerVersions.dhall | 46 +++-- buildkite/src/Constants/Network.dhall | 32 +++- .../Release/ItnOrchestratorArtifact.dhall | 48 ----- .../Jobs/Release/LeaderboardArtifact.dhall | 54 ------ ...dhall => MinaArtifactBullseyeDevnet.dhall} | 0 ...aArtifactBullseyeDevnetInstrumented.dhall} | 0 ... MinaArtifactBullseyeDevnetLightnet.dhall} | 0 ...hall => MinaArtifactBullseyeMainnet.dhall} | 3 +- ...al.dhall => MinaArtifactFocalDevnet.dhall} | 0 ...l.dhall => MinaArtifactFocalMainnet.dhall} | 3 +- ...inaArtifactOnlyDebianBullseyeDevnet.dhall} | 0 .../MinaToolchainArtifactBullseye.dhall | 9 +- .../Release/MinaToolchainArtifactFocal.dhall | 13 +- .../src/Jobs/Test/CheckGraphQLSchema.dhall | 3 + .../src/Jobs/Test/ConnectToBerkeley.dhall | 51 ----- buildkite/src/Jobs/Test/ConnectToDevnet.dhall | 1 + .../src/Jobs/Test/DevelopCompatibility.dhall | 14 +- buildkite/src/Jobs/Test/ReplayerTest.dhall | 2 +- .../Jobs/Test/RosettaIntegrationTests.dhall | 9 +- .../Test/RosettaIntegrationTestsLong.dhall | 7 +- .../src/Jobs/Test/RunSnarkProfiler.dhall | 3 + buildkite/src/Jobs/Test/SingleNodeTest.dhall | 4 + .../Jobs/Test/TestnetIntegrationTests.dhall | 4 +- .../Test/TestnetIntegrationTestsLong.dhall | 4 +- buildkite/src/Jobs/Test/VersionLint.dhall | 12 +- scripts/docker/helper.sh | 2 +- 35 files changed, 305 insertions(+), 430 deletions(-) delete mode 100644 buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall delete mode 100644 buildkite/src/Jobs/Release/LeaderboardArtifact.dhall rename buildkite/src/Jobs/Release/{MinaArtifactBullseye.dhall => MinaArtifactBullseyeDevnet.dhall} (100%) rename buildkite/src/Jobs/Release/{MinaArtifactBullseyeInstrumented.dhall => MinaArtifactBullseyeDevnetInstrumented.dhall} (100%) rename buildkite/src/Jobs/Release/{MinaArtifactBullseyeLightnet.dhall => MinaArtifactBullseyeDevnetLightnet.dhall} (100%) rename buildkite/src/Jobs/Release/{MinaArtifactMainnetBullseye.dhall => MinaArtifactBullseyeMainnet.dhall} (88%) rename buildkite/src/Jobs/Release/{MinaArtifactFocal.dhall => MinaArtifactFocalDevnet.dhall} (100%) rename buildkite/src/Jobs/Release/{MinaArtifactMainnetFocal.dhall => MinaArtifactFocalMainnet.dhall} (89%) rename buildkite/src/Jobs/Release/{MinaArtifactOnlyDebianBullseye.dhall => MinaArtifactOnlyDebianBullseyeDevnet.dhall} (100%) delete mode 100644 buildkite/src/Jobs/Test/ConnectToBerkeley.dhall diff --git a/buildkite/src/Command/DockerImage.dhall b/buildkite/src/Command/DockerImage.dhall index fc0dd9165fe..917a50fb621 100644 --- a/buildkite/src/Command/DockerImage.dhall +++ b/buildkite/src/Command/DockerImage.dhall @@ -10,8 +10,6 @@ let Size = ./Size.dhall let Profiles = ../Constants/Profiles.dhall -let Artifacts = ../Constants/Artifacts.dhall - let BuildFlags = ../Constants/BuildFlags.dhall let Cmd = ../Lib/Cmds.dhall @@ -20,59 +18,81 @@ let DockerLogin = ../Command/DockerLogin/Type.dhall let DebianRepo = ../Constants/DebianRepo.dhall +let DebianVersions = ../Constants/DebianVersions.dhall + let Network = ../Constants/Network.dhall +let Artifacts = ../Constants/Artifacts.dhall + let ReleaseSpec = { Type = { deps : List Command.TaggedKey.Type , network : Text - , service : Text + , service : Artifacts.Type , version : Text , branch : Text , repo : Text , no_cache : Bool - , deb_codename : Text + , deb_codename : DebianVersions.DebVersion , deb_release : Text , deb_version : Text , deb_profile : Profiles.Type , deb_repo : DebianRepo.Type , build_flags : BuildFlags.Type - , step_key : Text + , step_key_suffix : Text , if : Optional B/If } , default = { deps = [] : List Command.TaggedKey.Type , network = "${Network.lowerName Network.Type.Devnet}" , version = "\\\${MINA_DOCKER_TAG}" - , service = Artifacts.dockerName Artifacts.Type.Daemon + , service = Artifacts.Type.Daemon , branch = "\\\${BUILDKITE_BRANCH}" , repo = "\\\${BUILDKITE_REPO}" - , deb_codename = "bullseye" + , deb_codename = DebianVersions.DebVersion.Bullseye , deb_release = "\\\${MINA_DEB_RELEASE}" , deb_version = "\\\${MINA_DEB_VERSION}" , deb_profile = Profiles.Type.Standard , build_flags = BuildFlags.Type.None , deb_repo = DebianRepo.Type.PackagesO1Test , no_cache = False - , step_key = "daemon-standard-docker-image" + , step_key_suffix = "-docker-image" , if = None B/If } } +let stepKey = + \(spec : ReleaseSpec.Type) + -> "${Artifacts.lowerName + spec.service}${Profiles.toLabelSegment + spec.deb_profile}${BuildFlags.toLabelSegment + spec.build_flags}${spec.step_key_suffix}" + +let stepLabel = + \(spec : ReleaseSpec.Type) + -> "Docker: ${Artifacts.capitalName + spec.service} ${spec.network} ${DebianVersions.capitalName + spec.deb_codename} ${Profiles.toSuffixUppercase + spec.deb_profile} ${BuildFlags.toSuffixUppercase + spec.build_flags}" + let generateStep = \(spec : ReleaseSpec.Type) - -> let exportMinaDebCmd = "export MINA_DEB_CODENAME=${spec.deb_codename}" + -> let exportMinaDebCmd = + "export MINA_DEB_CODENAME=${DebianVersions.lowerName + spec.deb_codename}" let maybeCacheOption = if spec.no_cache then "--no-cache" else "" let buildDockerCmd = "./scripts/docker/build.sh" - ++ " --service ${spec.service}" + ++ " --service ${Artifacts.dockerName spec.service}" ++ " --network ${spec.network}" ++ " --version ${spec.version}" ++ " --branch ${spec.branch}" ++ " ${maybeCacheOption} " - ++ " --deb-codename ${spec.deb_codename}" + ++ " --deb-codename ${DebianVersions.lowerName + spec.deb_codename}" ++ " --deb-repo ${DebianRepo.address spec.deb_repo}" ++ " --deb-release ${spec.deb_release}" ++ " --deb-version ${spec.deb_version}" @@ -83,10 +103,11 @@ let generateStep = let releaseDockerCmd = "./scripts/docker/release.sh" - ++ " --service ${spec.service}" + ++ " --service ${Artifacts.dockerName spec.service}" ++ " --version ${spec.version}" ++ " --network ${spec.network}" - ++ " --deb-codename ${spec.deb_codename}" + ++ " --deb-codename ${DebianVersions.lowerName + spec.deb_codename}" ++ " --deb-version ${spec.deb_version}" ++ " --deb-profile ${Profiles.lowerName spec.deb_profile}" ++ " --deb-build-flags ${BuildFlags.lowerName @@ -123,12 +144,16 @@ let generateStep = in Command.build Command.Config::{ , commands = commands - , label = "Docker: ${spec.step_key}" - , key = spec.step_key + , label = "${stepLabel spec}" + , key = "${stepKey spec}" , target = Size.XLarge , docker_login = Some DockerLogin::{=} , depends_on = spec.deps , if = spec.if } -in { generateStep = generateStep, ReleaseSpec = ReleaseSpec } +in { generateStep = generateStep + , ReleaseSpec = ReleaseSpec + , stepKey = stepKey + , stepLabel = stepLabel + } diff --git a/buildkite/src/Command/HardforkPackageGeneration.dhall b/buildkite/src/Command/HardforkPackageGeneration.dhall index 183826d81b4..a8b9edf84b7 100644 --- a/buildkite/src/Command/HardforkPackageGeneration.dhall +++ b/buildkite/src/Command/HardforkPackageGeneration.dhall @@ -98,6 +98,17 @@ let pipeline "gcr.io/o1labs-192920/mina-daemon:\${BUILDKITE_COMMIT:0:7}-${DebianVersions.lowerName debVersion}-${network_name}" + let dockerSpec = + DockerImage.ReleaseSpec::{ + , deps = + [ { name = pipelineName, key = generateLedgersJobKey } ] + , service = Artifacts.Type.Daemon + , network = network_name + , deb_codename = debVersion + , deb_profile = profile + , deb_repo = DebianRepo.Type.Local + } + in Pipeline.Config::{ , spec = JobSpec::{ , dirtyWhen = [ S.everything ] @@ -155,20 +166,7 @@ let pipeline , key = "publish-hardfork-deb-pkg" , target = Size.Small } - , DockerImage.generateStep - DockerImage.ReleaseSpec::{ - , deps = - [ { name = pipelineName, key = generateLedgersJobKey } ] - , service = Artifacts.dockerName Artifacts.Type.Daemon - , network = network_name - , deb_codename = "${DebianVersions.lowerName debVersion}" - , deb_profile = profile - , deb_repo = DebianRepo.Type.Local - , step_key = - "daemon-berkeley-${DebianVersions.lowerName - debVersion}${Profiles.toLabelSegment - profile}-docker-image" - } + , DockerImage.generateStep dockerSpec , Command.build Command.Config::{ , commands = @@ -182,10 +180,7 @@ let pipeline , target = Size.XLarge , depends_on = [ { name = pipelineName - , key = - "daemon-berkeley-${DebianVersions.lowerName - debVersion}${Profiles.toLabelSegment - profile}-docker-image" + , key = DockerImage.stepKey dockerSpec } ] , if = None B/If @@ -202,10 +197,7 @@ let pipeline , target = Size.XLarge , depends_on = [ { name = pipelineName - , key = - "daemon-berkeley-${DebianVersions.lowerName - debVersion}${Profiles.toLabelSegment - profile}-docker-image" + , key = DockerImage.stepKey dockerSpec } ] , if = None B/If @@ -214,28 +206,20 @@ let pipeline DockerImage.ReleaseSpec::{ , deps = [ { name = pipelineName, key = generateLedgersJobKey } ] - , service = Artifacts.dockerName Artifacts.Type.Archive + , service = Artifacts.Type.Archive , network = network_name - , deb_codename = "${DebianVersions.lowerName debVersion}" + , deb_codename = debVersion , deb_profile = profile , deb_repo = DebianRepo.Type.Local - , step_key = - "archive-${DebianVersions.lowerName - debVersion}${Profiles.toLabelSegment - profile}-docker-image" } , DockerImage.generateStep DockerImage.ReleaseSpec::{ , deps = [ { name = pipelineName, key = generateLedgersJobKey } ] - , service = Artifacts.dockerName Artifacts.Type.Rosetta + , service = Artifacts.Type.Rosetta , network = network_name , deb_repo = DebianRepo.Type.Local - , deb_codename = "${DebianVersions.lowerName debVersion}" - , step_key = - "rosetta-${DebianVersions.lowerName - debVersion}${Profiles.toLabelSegment - profile}-docker-image" + , deb_codename = debVersion } ] } diff --git a/buildkite/src/Command/MinaArtifact.dhall b/buildkite/src/Command/MinaArtifact.dhall index e4f15a74db3..176e8380ae8 100644 --- a/buildkite/src/Command/MinaArtifact.dhall +++ b/buildkite/src/Command/MinaArtifact.dhall @@ -42,7 +42,7 @@ let MinaBuildSpec = , artifacts : List Artifacts.Type , debVersion : DebianVersions.DebVersion , profile : Profiles.Type - , networks : List Network.Type + , network : Network.Type , buildFlags : BuildFlags.Type , toolchainSelectMode : Toolchain.SelectionMode , mode : PipelineMode.Type @@ -55,7 +55,7 @@ let MinaBuildSpec = , debVersion = DebianVersions.DebVersion.Bullseye , profile = Profiles.Type.Standard , buildFlags = BuildFlags.Type.None - , networks = [ Network.Type.Devnet ] + , network = Network.Type.Devnet , toolchainSelectMode = Toolchain.SelectionMode.ByDebian , mode = PipelineMode.Type.PullRequest , tags = [ PipelineTag.Type.Long, PipelineTag.Type.Release ] @@ -63,6 +63,30 @@ let MinaBuildSpec = } } +let labelSuffix + : MinaBuildSpec.Type -> Text + = \(spec : MinaBuildSpec.Type) + -> "${DebianVersions.capitalName + spec.debVersion} ${Network.capitalName + spec.network} ${Profiles.toSuffixUppercase + spec.profile} ${BuildFlags.toSuffixUppercase + spec.buildFlags}" + +let nameSuffix + : MinaBuildSpec.Type -> Text + = \(spec : MinaBuildSpec.Type) + -> "${DebianVersions.capitalName + spec.debVersion}${Network.capitalName + spec.network}${Profiles.toSuffixUppercase + spec.profile}${BuildFlags.toSuffixUppercase + spec.buildFlags}" + +let keySuffix + : MinaBuildSpec.Type -> Text + = \(spec : MinaBuildSpec.Type) + -> "${Profiles.toLabelSegment spec.profile}${BuildFlags.toLabelSegment + spec.buildFlags}" + let build_artifacts : MinaBuildSpec.Type -> Command.Type = \(spec : MinaBuildSpec.Type) @@ -79,22 +103,18 @@ let build_artifacts , "MINA_COMMIT_SHA1=\$BUILDKITE_COMMIT" , "MINA_DEB_CODENAME=${DebianVersions.lowerName spec.debVersion}" - , Network.foldMinaBuildMainnetEnv spec.networks + , Network.buildMainnetEnv spec.network ] # BuildFlags.buildEnvs spec.buildFlags ) "./buildkite/scripts/build-release.sh ${Artifacts.toDebianNames spec.artifacts - spec.networks}" + spec.network}" # [ Cmd.run "./buildkite/scripts/debian/upload-to-gs.sh ${DebianVersions.lowerName spec.debVersion}" ] - , label = - "Build Mina for ${DebianVersions.capitalName - spec.debVersion} ${Profiles.toSuffixUppercase - spec.profile} ${BuildFlags.toSuffixUppercase - spec.buildFlags}" + , label = "Debian: Build ${labelSuffix spec}" , key = "build-deb-pkg" , target = Size.XLarge , retries = @@ -115,124 +135,97 @@ let docker_step DebianVersions.dependsOnStep (Some spec.prefix) spec.debVersion + spec.network spec.profile spec.buildFlags step_dep_name in merge { Daemon = - Prelude.List.map - Network.Type - DockerImage.ReleaseSpec.Type - ( \(n : Network.Type) - -> DockerImage.ReleaseSpec::{ - , deps = deps - , service = - Artifacts.dockerName Artifacts.Type.Daemon - , network = Network.lowerName n - , deb_codename = - "${DebianVersions.lowerName spec.debVersion}" - , deb_profile = spec.profile - , build_flags = spec.buildFlags - , deb_repo = DebianRepo.Type.Local - , step_key = - "daemon-${Network.lowerName - n}-${DebianVersions.lowerName - spec.debVersion}${Profiles.toLabelSegment - spec.profile}${BuildFlags.toLabelSegment - spec.buildFlags}-docker-image" - } - ) - spec.networks + [ DockerImage.ReleaseSpec::{ + , deps = deps + , service = Artifacts.Type.Daemon + , network = Network.lowerName spec.network + , deb_codename = spec.debVersion + , deb_profile = spec.profile + , build_flags = spec.buildFlags + , deb_repo = DebianRepo.Type.Local + } + ] , TestExecutive = [] : List DockerImage.ReleaseSpec.Type , LogProc = [] : List DockerImage.ReleaseSpec.Type + , Toolchain = + [ DockerImage.ReleaseSpec::{ + , service = Artifacts.Type.Toolchain + , network = Network.lowerName Network.Type.Devnet + , deb_codename = spec.debVersion + , deb_profile = spec.profile + , build_flags = spec.buildFlags + , deb_repo = DebianRepo.Type.Local + } + ] + , ItnOrchestrator = + [ DockerImage.ReleaseSpec::{ + , service = Artifacts.Type.ItnOrchestrator + , network = Network.lowerName Network.Type.Devnet + , deb_repo = DebianRepo.Type.Local + } + ] + , Leaderboard = + [ DockerImage.ReleaseSpec::{ + , service = Artifacts.Type.Leaderboard + } + ] , BatchTxn = [ DockerImage.ReleaseSpec::{ , deps = deps - , service = Artifacts.dockerName Artifacts.Type.BatchTxn + , service = Artifacts.Type.BatchTxn , network = Network.lowerName Network.Type.Devnet - , deb_codename = - "${DebianVersions.lowerName spec.debVersion}" + , deb_codename = spec.debVersion , deb_profile = spec.profile , build_flags = spec.buildFlags , deb_repo = DebianRepo.Type.Local - , step_key = - "batch-txn-${DebianVersions.lowerName - spec.debVersion}${BuildFlags.toLabelSegment - spec.buildFlags}-docker-image" } ] , Archive = [ DockerImage.ReleaseSpec::{ , deps = deps - , service = Artifacts.dockerName Artifacts.Type.Archive - , deb_codename = - "${DebianVersions.lowerName spec.debVersion}" + , service = Artifacts.Type.Archive + , deb_codename = spec.debVersion , deb_profile = spec.profile , build_flags = spec.buildFlags , deb_repo = DebianRepo.Type.Local - , step_key = - "archive-${DebianVersions.lowerName - spec.debVersion}${Profiles.toLabelSegment - spec.profile}${BuildFlags.toLabelSegment - spec.buildFlags}-docker-image" } ] , Rosetta = - Prelude.List.map - Network.Type - DockerImage.ReleaseSpec.Type - ( \(n : Network.Type) - -> DockerImage.ReleaseSpec::{ - , deps = deps - , service = - Artifacts.dockerName Artifacts.Type.Rosetta - , network = Network.lowerName n - , deb_codename = - "${DebianVersions.lowerName spec.debVersion}" - , deb_profile = spec.profile - , deb_repo = DebianRepo.Type.Local - , step_key = - "rosetta-${Network.lowerName - n}-${DebianVersions.lowerName - spec.debVersion}${Profiles.toLabelSegment - spec.profile}-docker-image" - } - ) - spec.networks + [ DockerImage.ReleaseSpec::{ + , deps = deps + , service = Artifacts.Type.Rosetta + , network = Network.lowerName spec.network + , deb_codename = spec.debVersion + , deb_profile = spec.profile + , deb_repo = DebianRepo.Type.Local + } + ] , ZkappTestTransaction = [ DockerImage.ReleaseSpec::{ , deps = deps - , service = - Artifacts.dockerName Artifacts.Type.ZkappTestTransaction + , service = Artifacts.Type.ZkappTestTransaction , build_flags = spec.buildFlags , deb_repo = DebianRepo.Type.Local , deb_profile = spec.profile - , deb_codename = - "${DebianVersions.lowerName spec.debVersion}" - , step_key = - "zkapp-test-transaction-${DebianVersions.lowerName - spec.debVersion}${Profiles.toLabelSegment - spec.profile}${BuildFlags.toLabelSegment - spec.buildFlags}-docker-image" + , deb_codename = spec.debVersion } ] , FunctionalTestSuite = [ DockerImage.ReleaseSpec::{ , deps = deps - , service = - Artifacts.dockerName Artifacts.Type.FunctionalTestSuite + , service = Artifacts.Type.FunctionalTestSuite , network = Network.lowerName Network.Type.Devnet - , deb_codename = - "${DebianVersions.lowerName spec.debVersion}" + , deb_codename = spec.debVersion , build_flags = spec.buildFlags , deb_repo = DebianRepo.Type.Local , deb_profile = spec.profile - , step_key = - "test-suite-${DebianVersions.lowerName - spec.debVersion}${Profiles.toLabelSegment - spec.profile}${BuildFlags.toLabelSegment - spec.buildFlags}-docker-image" } ] } @@ -282,15 +275,13 @@ let publish_to_debian_repo = , "MINA_DEB_RELEASE=${DebianChannel.lowerName spec.channel}" ] "./buildkite/scripts/debian/publish.sh" - , label = - "Publish Mina for ${DebianVersions.capitalName - spec.debVersion} ${Profiles.toSuffixUppercase - spec.profile}" + , label = "Debian: Publish ${labelSuffix spec}" , key = "publish-deb-pkg" , depends_on = DebianVersions.dependsOnStep (Some spec.prefix) spec.debVersion + spec.network spec.profile spec.buildFlags "build" @@ -305,11 +296,7 @@ let pipelineBuilder , spec = JobSpec::{ , dirtyWhen = DebianVersions.dirtyWhen spec.debVersion , path = "Release" - , name = - "${spec.prefix}${DebianVersions.capitalName - spec.debVersion}${Profiles.toSuffixUppercase - spec.profile}${BuildFlags.toSuffixUppercase - spec.buildFlags}" + , name = "${spec.prefix}${nameSuffix spec}" , tags = spec.tags , mode = spec.mode } @@ -341,4 +328,6 @@ let pipeline in { pipeline = pipeline , onlyDebianPipeline = onlyDebianPipeline , MinaBuildSpec = MinaBuildSpec + , labelSuffix = labelSuffix + , keySuffix = keySuffix } diff --git a/buildkite/src/Command/Promotion/PromoteDebian.dhall b/buildkite/src/Command/Promotion/PromoteDebian.dhall index 0019779bbba..2ac01eb006a 100644 --- a/buildkite/src/Command/Promotion/PromoteDebian.dhall +++ b/buildkite/src/Command/Promotion/PromoteDebian.dhall @@ -40,7 +40,7 @@ let PromoteDebianSpec = , version = "\\\\\$MINA_DEB_VERSION" , new_version = "\\\\\$MINA_DEB_VERSION" , architecture = "amd64" - , network = Network.Type.Berkeley + , network = Network.Type.Devnet , codename = DebianVersions.DebVersion.Bullseye , from_channel = DebianChannel.Type.Unstable , to_channel = DebianChannel.Type.NightlyCompatible diff --git a/buildkite/src/Command/Promotion/PromoteDocker.dhall b/buildkite/src/Command/Promotion/PromoteDocker.dhall index cbc60627dc8..7e56225d5ef 100644 --- a/buildkite/src/Command/Promotion/PromoteDocker.dhall +++ b/buildkite/src/Command/Promotion/PromoteDocker.dhall @@ -41,7 +41,7 @@ let PromoteDockerSpec = , new_tags = [] : List Text , step_key = "promote-docker" , profile = Profiles.Type.Standard - , network = Network.Type.Berkeley + , network = Network.Type.Devnet , codename = DebianVersions.DebVersion.Bullseye , if = None B/If , publish = False diff --git a/buildkite/src/Command/ReplayerTest.dhall b/buildkite/src/Command/ReplayerTest.dhall index 6a742c7e7fd..d93f70a2677 100644 --- a/buildkite/src/Command/ReplayerTest.dhall +++ b/buildkite/src/Command/ReplayerTest.dhall @@ -17,7 +17,7 @@ in { step = ([] : List Text) "./src/test/archive/sample_db/archive_db.sql" Artifacts.Type.Archive - (None Network.Type) + Network.Type.Devnet "./buildkite/scripts/replayer-test.sh" ] , label = "Archive: Replayer test" diff --git a/buildkite/src/Command/RunWithPostgres.dhall b/buildkite/src/Command/RunWithPostgres.dhall index 5269fda5619..a9ad935999e 100644 --- a/buildkite/src/Command/RunWithPostgres.dhall +++ b/buildkite/src/Command/RunWithPostgres.dhall @@ -2,10 +2,6 @@ let Prelude = ../External/Prelude.dhall let P = Prelude -let Optional/map = Prelude.Optional.map - -let Optional/default = Prelude.Optional.default - let Text/concatMap = P.Text.concatMap let Cmd = ../Lib/Cmds.dhall @@ -17,16 +13,11 @@ let Artifacts = ../Constants/Artifacts.dhall let Network = ../Constants/Network.dhall let runInDockerWithPostgresConn - : List Text - -> Text - -> Artifacts.Type - -> Optional Network.Type - -> Text - -> Cmd.Type + : List Text -> Text -> Artifacts.Type -> Network.Type -> Text -> Cmd.Type = \(environment : List Text) -> \(initScript : Text) -> \(docker : Artifacts.Type) - -> \(network : Optional Network.Type) + -> \(network : Network.Type) -> \(innerScript : Text) -> let port = "5432" @@ -40,15 +31,6 @@ let runInDockerWithPostgresConn let dbName = "archive" - let maybeNetwork = - Optional/map - Network.Type - Text - (\(network : Network.Type) -> "-${Network.lowerName network}") - network - - let networkOrDefault = Optional/default Text "" maybeNetwork - let pg_conn = "postgres://${user}:${password}@localhost:${port}/${dbName}" @@ -67,7 +49,7 @@ let runInDockerWithPostgresConn let outerDir : Text - = "\\\$BUILDKITE_BUILD_CHECKOUT_PATH${networkOrDefault}" + = "\\\$BUILDKITE_BUILD_CHECKOUT_PATH" let minaDockerTag : Text diff --git a/buildkite/src/Constants/Artifacts.dhall b/buildkite/src/Constants/Artifacts.dhall index b41f52cf28c..673b14c3808 100644 --- a/buildkite/src/Constants/Artifacts.dhall +++ b/buildkite/src/Constants/Artifacts.dhall @@ -18,6 +18,9 @@ let Artifact | Rosetta | ZkappTestTransaction | FunctionalTestSuite + | Toolchain + | ItnOrchestrator + | Leaderboard > let AllButTests = @@ -33,7 +36,13 @@ let AllButTests = let Main = [ Artifact.Daemon, Artifact.LogProc, Artifact.Archive, Artifact.Rosetta ] -let All = AllButTests # [ Artifact.FunctionalTestSuite ] +let All = + AllButTests + # [ Artifact.FunctionalTestSuite + , Artifact.Toolchain + , Artifact.ItnOrchestrator + , Artifact.Leaderboard + ] let capitalName = \(artifact : Artifact) @@ -46,6 +55,9 @@ let capitalName = , Rosetta = "Rosetta" , ZkappTestTransaction = "ZkappTestTransaction" , FunctionalTestSuite = "FunctionalTestSuite" + , Toolchain = "Toolchain" + , ItnOrchestrator = "ItnOrchestrator" + , Leaderboard = "Leaderboard" } artifact @@ -60,6 +72,9 @@ let lowerName = , Rosetta = "rosetta" , ZkappTestTransaction = "zkapp_test_transaction" , FunctionalTestSuite = "functional_test_suite" + , Toolchain = "toolchain" + , ItnOrchestrator = "itnOrchestrator" + , Leaderboard = "leaderboard" } artifact @@ -74,6 +89,9 @@ let dockerName = , Rosetta = "mina-rosetta" , ZkappTestTransaction = "mina-zkapp-test-transaction" , FunctionalTestSuite = "mina-test-suite" + , Toolchain = "mina-toolchain" + , ItnOrchestrator = "itn-orchestrator" + , Leaderboard = "leaderboard" } artifact @@ -89,38 +107,32 @@ let toDebianName = , Rosetta = "rosetta_${Network.lowerName network}" , ZkappTestTransaction = "zkapp_test_transaction" , FunctionalTestSuite = "functional_test_suite" + , Toolchain = "" + , ItnOrchestrator = "" + , Leaderboard = "" } artifact let toDebianNames = \(artifacts : List Artifact) - -> \(networks : List Network.Type) + -> \(network : Network.Type) -> let list_of_list_of_debians = Prelude.List.map Artifact (List Text) ( \(a : Artifact) -> merge - { Daemon = - Prelude.List.map - Network.Type - Text - (\(n : Network.Type) -> toDebianName a n) - networks + { Daemon = [ toDebianName a network ] , Archive = [ "archive" ] , LogProc = [ "logproc" ] , TestExecutive = [ "test_executive" ] , BatchTxn = [ "batch_txn" ] - , Rosetta = - Prelude.List.map - Network.Type - Text - ( \(n : Network.Type) - -> "rosetta_${Network.lowerName n}" - ) - networks + , Rosetta = [ toDebianName a network ] , ZkappTestTransaction = [ "zkapp_test_transaction" ] , FunctionalTestSuite = [ "functional_test_suite" ] + , Toolchain = [] : List Text + , ItnOrchestrator = [] : List Text + , Leaderboard = [] : List Text } a ) @@ -165,6 +177,9 @@ let dockerTag = "${version_and_codename}-${Network.lowerName network}" , ZkappTestTransaction = "${version_and_codename}" , FunctionalTestSuite = "${version_and_codename}" + , Toolchain = "${version_and_codename}" + , ItnOrchestrator = "${version_and_codename}" + , Leaderboard = "${version_and_codename}" } artifact diff --git a/buildkite/src/Constants/DebianVersions.dhall b/buildkite/src/Constants/DebianVersions.dhall index f3042f5f067..ba4bb08311b 100644 --- a/buildkite/src/Constants/DebianVersions.dhall +++ b/buildkite/src/Constants/DebianVersions.dhall @@ -4,6 +4,8 @@ let Optional/default = Prelude.Optional.default let Profiles = ./Profiles.dhall +let Network = ./Network.dhall + let BuildFlags = ./BuildFlags.dhall let S = ../Lib/SelectFiles.dhall @@ -33,6 +35,7 @@ let lowerName = let dependsOnStep = \(prefix : Optional Text) -> \(debVersion : DebVersion) + -> \(network : Network.Type) -> \(profile : Profiles.Type) -> \(buildFlag : BuildFlags.Type) -> \(step : Text) @@ -40,47 +43,28 @@ let dependsOnStep = let prefix = Optional/default Text "MinaArtifact" prefix + let name = + "${prefix}${capitalName + debVersion}${Network.capitalName + network}${profileSuffix}${BuildFlags.toSuffixUppercase + buildFlag}" + in merge - { Bookworm = - [ { name = - "${prefix}${profileSuffix}${BuildFlags.toSuffixUppercase - buildFlag}" - , key = "${step}-deb-pkg" - } - ] - , Bullseye = - [ { name = - "${prefix}${capitalName - debVersion}${profileSuffix}${BuildFlags.toSuffixUppercase - buildFlag}" - , key = "${step}-deb-pkg" - } - ] - , Jammy = - [ { name = - "${prefix}${capitalName - debVersion}${profileSuffix}${BuildFlags.toSuffixUppercase - buildFlag}" - , key = "${step}-deb-pkg" - } - ] - , Focal = - [ { name = - "${prefix}${capitalName - debVersion}${profileSuffix}${BuildFlags.toSuffixUppercase - buildFlag}" - , key = "${step}-deb-pkg" - } - ] + { Bookworm = [ { name = name, key = "${step}-deb-pkg" } ] + , Bullseye = [ { name = name, key = "${step}-deb-pkg" } ] + , Jammy = [ { name = name, key = "${step}-deb-pkg" } ] + , Focal = [ { name = name, key = "${step}-deb-pkg" } ] } debVersion let dependsOn = \(debVersion : DebVersion) + -> \(network : Network.Type) -> \(profile : Profiles.Type) -> dependsOnStep (None Text) debVersion + network profile BuildFlags.Type.None "build" diff --git a/buildkite/src/Constants/DockerVersions.dhall b/buildkite/src/Constants/DockerVersions.dhall index a09718f19e7..926a077b77d 100644 --- a/buildkite/src/Constants/DockerVersions.dhall +++ b/buildkite/src/Constants/DockerVersions.dhall @@ -1,9 +1,3 @@ -let Prelude = ../External/Prelude.dhall - -let Optional/map = Prelude.Optional.map - -let Optional/default = Prelude.Optional.default - let Profiles = ./Profiles.dhall let Artifacts = ./Artifacts.dhall @@ -36,43 +30,47 @@ let lowerName = let dependsOn = \(docker : Docker) - -> \(network : Optional Network.Type) + -> \(network : Network.Type) -> \(profile : Profiles.Type) -> \(binary : Artifacts.Type) - -> let profileSuffix = Profiles.toSuffixUppercase profile + -> let network = "${Network.capitalName network}" + + let profileSuffix = "${Profiles.toSuffixUppercase profile}" let prefix = "MinaArtifact" let suffix = "docker-image" - let maybeNetwork = - Optional/map - Network.Type - Text - (\(network : Network.Type) -> "-${Network.lowerName network}") - network - - let networkOrDefault = Optional/default Text "" maybeNetwork - - let key = - "${Artifacts.lowerName - binary}${networkOrDefault}-${lowerName docker}-${suffix}" + let key = "${Artifacts.lowerName binary}-${suffix}" in merge { Bookworm = - [ { name = "${prefix}${profileSuffix}", key = key } ] + [ { name = + "${prefix}${capitalName + docker}${network}${profileSuffix}" + , key = key + } + ] , Bullseye = - [ { name = "${prefix}${capitalName docker}${profileSuffix}" + [ { name = + "${prefix}${capitalName + docker}${network}${profileSuffix}" , key = key } ] , Jammy = - [ { name = "${prefix}${capitalName docker}${profileSuffix}" + [ { name = + "${prefix}${capitalName + docker}${network}${capitalName + docker}${profileSuffix}" , key = key } ] , Focal = - [ { name = "${prefix}${capitalName docker}${profileSuffix}" + [ { name = + "${prefix}${capitalName + docker}${network}${capitalName + docker}${profileSuffix}" , key = key } ] diff --git a/buildkite/src/Constants/Network.dhall b/buildkite/src/Constants/Network.dhall index 92462c87ae3..86094577017 100644 --- a/buildkite/src/Constants/Network.dhall +++ b/buildkite/src/Constants/Network.dhall @@ -4,23 +4,18 @@ let List/any = Prelude.List.any let Network : Type - = < Devnet | Mainnet | Berkeley > + = < Devnet | Mainnet > let capitalName = \(network : Network) - -> merge - { Devnet = "Devnet", Mainnet = "Mainnet", Berkeley = "Berkeley" } - network + -> merge { Devnet = "Devnet", Mainnet = "Mainnet" } network let lowerName = \(network : Network) - -> merge - { Devnet = "devnet", Mainnet = "mainnet", Berkeley = "berkeley" } - network + -> merge { Devnet = "devnet", Mainnet = "mainnet" } network let requiresMainnetBuild = - \(network : Network) - -> merge { Devnet = True, Mainnet = True, Berkeley = False } network + \(network : Network) -> merge { Devnet = False, Mainnet = True } network let foldMinaBuildMainnetEnv = \(networks : List Network) @@ -30,9 +25,28 @@ let foldMinaBuildMainnetEnv = else "MINA_BUILD_MAINNET=false" +let buildMainnetEnv = + \(network : Network) + -> if requiresMainnetBuild network + + then "MINA_BUILD_MAINNET=true" + + else "MINA_BUILD_MAINNET=false" + +let foldNames = + \(networks : List Network) + -> Prelude.List.fold + Network + networks + Text + (\(x : Network) -> \(y : Text) -> "${capitalName x}" ++ y) + "" + in { Type = Network , capitalName = capitalName , lowerName = lowerName , requiresMainnetBuild = requiresMainnetBuild , foldMinaBuildMainnetEnv = foldMinaBuildMainnetEnv + , buildMainnetEnv = buildMainnetEnv + , foldNames = foldNames } diff --git a/buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall b/buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall deleted file mode 100644 index 8e9d8271d88..00000000000 --- a/buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall +++ /dev/null @@ -1,48 +0,0 @@ -let S = ../../Lib/SelectFiles.dhall - -let Pipeline = ../../Pipeline/Dsl.dhall - -let PipelineTag = ../../Pipeline/Tag.dhall - -let JobSpec = ../../Pipeline/JobSpec.dhall - -let DockerImage = ../../Command/DockerImage.dhall - -let Profiles = ../../Constants/Profiles.dhall - -let DebianVersions = ../../Constants/DebianVersions.dhall - -let DebianRepo = ../../Constants/DebianRepo.dhall - -let Network = ../../Constants/Network.dhall - -let spec = - DockerImage.ReleaseSpec::{ - , service = "itn-orchestrator" - , step_key = "itn-orchestrator-docker-image" - , network = "${Network.lowerName Network.Type.Devnet}" - , deb_repo = DebianRepo.Type.Local - , deps = - DebianVersions.dependsOn - DebianVersions.DebVersion.Bullseye - Profiles.Type.Standard - } - -in Pipeline.build - Pipeline.Config::{ - , spec = JobSpec::{ - , dirtyWhen = - [ S.strictlyStart - (S.contains "buildkite/src/Jobs/Release/ItnOrchestratorArtifact") - , S.strictlyStart (S.contains "src/app/itn_orchestrator") - ] - , path = "Release" - , name = "ItnOrchestratorArtifact" - , tags = - [ PipelineTag.Type.Long - , PipelineTag.Type.Release - , PipelineTag.Type.Stable - ] - } - , steps = [ DockerImage.generateStep spec ] - } diff --git a/buildkite/src/Jobs/Release/LeaderboardArtifact.dhall b/buildkite/src/Jobs/Release/LeaderboardArtifact.dhall deleted file mode 100644 index 2482ee5b0d3..00000000000 --- a/buildkite/src/Jobs/Release/LeaderboardArtifact.dhall +++ /dev/null @@ -1,54 +0,0 @@ -let Cmd = ../../Lib/Cmds.dhall - -let S = ../../Lib/SelectFiles.dhall - -let Pipeline = ../../Pipeline/Dsl.dhall - -let PipelineTag = ../../Pipeline/Tag.dhall - -let JobSpec = ../../Pipeline/JobSpec.dhall - -let Command = ../../Command/Base.dhall - -let Size = ../../Command/Size.dhall - -let DockerImage = ../../Command/DockerImage.dhall - -let spec = - DockerImage.ReleaseSpec::{ - , service = "leaderboard" - , step_key = "leaderboard-docker-image" - } - -in Pipeline.build - Pipeline.Config::{ - , spec = JobSpec::{ - , dirtyWhen = - [ S.strictlyStart - (S.contains "buildkite/src/Jobs/Release/LeaderboardArtifact") - , S.strictlyStart (S.contains "frontend/leaderboard") - ] - , path = "Release" - , name = "LeaderboardArtifact" - , tags = - [ PipelineTag.Type.Long - , PipelineTag.Type.Release - , PipelineTag.Type.Stable - ] - } - , steps = - [ Command.build - Command.Config::{ - , commands = - [ Cmd.run - "echo export MINA_VERSION=\$(cat frontend/leaderboard/package.json | jq '.version') > LEADERBOARD_DEPLOY_ENV" - ] - , label = "Setup Leaderboard docker image deploy environment" - , key = "setup-deploy-env" - , target = Size.Small - , artifact_paths = - [ S.contains "frontend/leaderboard/package.json" ] - } - , DockerImage.generateStep spec - ] - } diff --git a/buildkite/src/Jobs/Release/MinaArtifactBullseye.dhall b/buildkite/src/Jobs/Release/MinaArtifactBullseyeDevnet.dhall similarity index 100% rename from buildkite/src/Jobs/Release/MinaArtifactBullseye.dhall rename to buildkite/src/Jobs/Release/MinaArtifactBullseyeDevnet.dhall diff --git a/buildkite/src/Jobs/Release/MinaArtifactBullseyeInstrumented.dhall b/buildkite/src/Jobs/Release/MinaArtifactBullseyeDevnetInstrumented.dhall similarity index 100% rename from buildkite/src/Jobs/Release/MinaArtifactBullseyeInstrumented.dhall rename to buildkite/src/Jobs/Release/MinaArtifactBullseyeDevnetInstrumented.dhall diff --git a/buildkite/src/Jobs/Release/MinaArtifactBullseyeLightnet.dhall b/buildkite/src/Jobs/Release/MinaArtifactBullseyeDevnetLightnet.dhall similarity index 100% rename from buildkite/src/Jobs/Release/MinaArtifactBullseyeLightnet.dhall rename to buildkite/src/Jobs/Release/MinaArtifactBullseyeDevnetLightnet.dhall diff --git a/buildkite/src/Jobs/Release/MinaArtifactMainnetBullseye.dhall b/buildkite/src/Jobs/Release/MinaArtifactBullseyeMainnet.dhall similarity index 88% rename from buildkite/src/Jobs/Release/MinaArtifactMainnetBullseye.dhall rename to buildkite/src/Jobs/Release/MinaArtifactBullseyeMainnet.dhall index aa4037d8626..18633217d3c 100644 --- a/buildkite/src/Jobs/Release/MinaArtifactMainnetBullseye.dhall +++ b/buildkite/src/Jobs/Release/MinaArtifactBullseyeMainnet.dhall @@ -21,13 +21,12 @@ in Pipeline.build , Artifacts.Type.Rosetta , Artifacts.Type.ZkappTestTransaction ] - , networks = [ Network.Type.Devnet, Network.Type.Mainnet ] + , network = Network.Type.Mainnet , tags = [ PipelineTag.Type.Long , PipelineTag.Type.Release , PipelineTag.Type.Stable ] , mode = PipelineMode.Type.Stable - , prefix = "MinaArtifactMainnet" } ) diff --git a/buildkite/src/Jobs/Release/MinaArtifactFocal.dhall b/buildkite/src/Jobs/Release/MinaArtifactFocalDevnet.dhall similarity index 100% rename from buildkite/src/Jobs/Release/MinaArtifactFocal.dhall rename to buildkite/src/Jobs/Release/MinaArtifactFocalDevnet.dhall diff --git a/buildkite/src/Jobs/Release/MinaArtifactMainnetFocal.dhall b/buildkite/src/Jobs/Release/MinaArtifactFocalMainnet.dhall similarity index 89% rename from buildkite/src/Jobs/Release/MinaArtifactMainnetFocal.dhall rename to buildkite/src/Jobs/Release/MinaArtifactFocalMainnet.dhall index 82d9d49a532..76829c42836 100644 --- a/buildkite/src/Jobs/Release/MinaArtifactMainnetFocal.dhall +++ b/buildkite/src/Jobs/Release/MinaArtifactFocalMainnet.dhall @@ -24,13 +24,12 @@ in Pipeline.build , Artifacts.Type.ZkappTestTransaction ] , debVersion = DebianVersions.DebVersion.Focal - , networks = [ Network.Type.Devnet, Network.Type.Mainnet ] + , network = Network.Type.Mainnet , tags = [ PipelineTag.Type.Long , PipelineTag.Type.Release , PipelineTag.Type.Stable ] , mode = PipelineMode.Type.Stable - , prefix = "MinaArtifactMainnet" } ) diff --git a/buildkite/src/Jobs/Release/MinaArtifactOnlyDebianBullseye.dhall b/buildkite/src/Jobs/Release/MinaArtifactOnlyDebianBullseyeDevnet.dhall similarity index 100% rename from buildkite/src/Jobs/Release/MinaArtifactOnlyDebianBullseye.dhall rename to buildkite/src/Jobs/Release/MinaArtifactOnlyDebianBullseyeDevnet.dhall diff --git a/buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall b/buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall index 86e483e1e31..357da774d05 100644 --- a/buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall +++ b/buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall @@ -8,6 +8,10 @@ let JobSpec = ../../Pipeline/JobSpec.dhall let DockerImage = ../../Command/DockerImage.dhall +let DebianVersions = ../../Constants/DebianVersions.dhall + +let Artifacts = ../../Constants/Artifacts.dhall + in Pipeline.build Pipeline.Config::{ , spec = JobSpec::{ @@ -27,10 +31,9 @@ in Pipeline.build , steps = [ let toolchainBullseyeSpec = DockerImage.ReleaseSpec::{ - , service = "mina-toolchain" - , deb_codename = "bullseye" + , service = Artifacts.Type.Toolchain + , deb_codename = DebianVersions.DebVersion.Bullseye , no_cache = True - , step_key = "toolchain-bullseye-docker-image" } in DockerImage.generateStep toolchainBullseyeSpec diff --git a/buildkite/src/Jobs/Release/MinaToolchainArtifactFocal.dhall b/buildkite/src/Jobs/Release/MinaToolchainArtifactFocal.dhall index ab0030f99f4..2ffe0472892 100644 --- a/buildkite/src/Jobs/Release/MinaToolchainArtifactFocal.dhall +++ b/buildkite/src/Jobs/Release/MinaToolchainArtifactFocal.dhall @@ -8,6 +8,10 @@ let JobSpec = ../../Pipeline/JobSpec.dhall let DockerImage = ../../Command/DockerImage.dhall +let DebianVersions = ../../Constants/DebianVersions.dhall + +let Artifacts = ../../Constants/Artifacts.dhall + in Pipeline.build Pipeline.Config::{ , spec = JobSpec::{ @@ -25,14 +29,13 @@ in Pipeline.build , tags = [ PipelineTag.Type.Toolchain ] } , steps = - [ let toolchainSpec = + [ let toolchainBullseyeSpec = DockerImage.ReleaseSpec::{ - , service = "mina-toolchain" - , deb_codename = "focal" + , service = Artifacts.Type.Toolchain + , deb_codename = DebianVersions.DebVersion.Focal , no_cache = True - , step_key = "toolchain-focal-docker-image" } - in DockerImage.generateStep toolchainSpec + in DockerImage.generateStep toolchainBullseyeSpec ] } diff --git a/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall b/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall index dee29949cd5..bef4ebb1e47 100644 --- a/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall +++ b/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall @@ -12,9 +12,12 @@ let DebianVersions = ../../Constants/DebianVersions.dhall let Profiles = ../../Constants/Profiles.dhall +let Network = ../../Constants/Network.dhall + let dependsOn = DebianVersions.dependsOn DebianVersions.DebVersion.Bullseye + Network.Type.Devnet Profiles.Type.Standard in Pipeline.build diff --git a/buildkite/src/Jobs/Test/ConnectToBerkeley.dhall b/buildkite/src/Jobs/Test/ConnectToBerkeley.dhall deleted file mode 100644 index 312e63f9435..00000000000 --- a/buildkite/src/Jobs/Test/ConnectToBerkeley.dhall +++ /dev/null @@ -1,51 +0,0 @@ -let S = ../../Lib/SelectFiles.dhall - -let B = ../../External/Buildkite.dhall - -let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type - -let JobSpec = ../../Pipeline/JobSpec.dhall - -let Pipeline = ../../Pipeline/Dsl.dhall - -let PipelineTag = ../../Pipeline/Tag.dhall - -let ConnectToTestnet = ../../Command/ConnectToTestnet.dhall - -let Profiles = ../../Constants/Profiles.dhall - -let Artifacts = ../../Constants/Artifacts.dhall - -let Network = ../../Constants/Network.dhall - -let Dockers = ../../Constants/DockerVersions.dhall - -let dependsOn = - Dockers.dependsOn - Dockers.Type.Bullseye - (Some Network.Type.Berkeley) - Profiles.Type.Standard - Artifacts.Type.Daemon - -in Pipeline.build - Pipeline.Config::{ - , spec = JobSpec::{ - , dirtyWhen = - [ S.strictlyStart (S.contains "src") - , S.exactly "buildkite/scripts/connect-to-testnet" "sh" - , S.exactly "buildkite/src/Jobs/Test/ConnectToBerkeley" "dhall" - , S.exactly "buildkite/src/Command/ConnectToTestnet" "dhall" - ] - , path = "Test" - , name = "ConnectToBerkeley" - , tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] - } - , steps = - [ ConnectToTestnet.step - dependsOn - "berkeley" - "40s" - "2m" - (B/SoftFail.Boolean True) - ] - } diff --git a/buildkite/src/Jobs/Test/ConnectToDevnet.dhall b/buildkite/src/Jobs/Test/ConnectToDevnet.dhall index 97f05e0e19f..526ee022143 100644 --- a/buildkite/src/Jobs/Test/ConnectToDevnet.dhall +++ b/buildkite/src/Jobs/Test/ConnectToDevnet.dhall @@ -21,6 +21,7 @@ let Network = ../../Constants/Network.dhall let dependsOn = DebianVersions.dependsOn DebianVersions.DebVersion.Bullseye + Network.Type.Devnet Profiles.Type.Standard in Pipeline.build diff --git a/buildkite/src/Jobs/Test/DevelopCompatibility.dhall b/buildkite/src/Jobs/Test/DevelopCompatibility.dhall index 8660fdfa790..8bcecf6c27f 100644 --- a/buildkite/src/Jobs/Test/DevelopCompatibility.dhall +++ b/buildkite/src/Jobs/Test/DevelopCompatibility.dhall @@ -14,11 +14,17 @@ let Docker = ../../Command/Docker/Type.dhall let Size = ../../Command/Size.dhall +let DebianVersions = ../../Constants/DebianVersions.dhall + +let Network = ../../Constants/Network.dhall + +let Profiles = ../../Constants/Profiles.dhall + let dependsOn = - [ { name = "MinaArtifactBullseye" - , key = "daemon-devnet-bullseye-docker-image" - } - ] + DebianVersions.dependsOn + DebianVersions.DebVersion.Bullseye + Network.Type.Devnet + Profiles.Type.Lightnet in Pipeline.build Pipeline.Config::{ diff --git a/buildkite/src/Jobs/Test/ReplayerTest.dhall b/buildkite/src/Jobs/Test/ReplayerTest.dhall index 0e3d665e2ce..24ce7acf7e1 100644 --- a/buildkite/src/Jobs/Test/ReplayerTest.dhall +++ b/buildkite/src/Jobs/Test/ReplayerTest.dhall @@ -19,7 +19,7 @@ let Artifacts = ../../Constants/Artifacts.dhall let dependsOn = Dockers.dependsOn Dockers.Type.Bullseye - (None Network.Type) + Network.Type.Devnet Profiles.Type.Standard Artifacts.Type.Archive diff --git a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall index a336603b2a1..02ed47087de 100644 --- a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall +++ b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall @@ -26,6 +26,8 @@ let Artifacts = ../../Constants/Artifacts.dhall let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type +let network = Network.Type.Devnet + let dirtyWhen = [ S.strictlyStart (S.contains "src") , S.exactly "buildkite/src/Jobs/Test/RosettaIntegrationTests" "dhall" @@ -55,12 +57,13 @@ in Pipeline.build ([] : List Text) "./src/test/archive/sample_db/archive_db.sql" Artifacts.Type.Rosetta - (Some Network.Type.Berkeley) + Network.Type.Devnet "./buildkite/scripts/rosetta-indexer-test.sh" , Cmd.runInDocker Cmd.Docker::{ , image = - "gcr.io/o1labs-192920/mina-rosetta:\\\${MINA_DOCKER_TAG}-berkeley" + "gcr.io/o1labs-192920/mina-rosetta:\\\${MINA_DOCKER_TAG}-${Network.lowerName + network}" } "buildkite/scripts/rosetta-integration-tests-fast.sh" ] @@ -71,7 +74,7 @@ in Pipeline.build , depends_on = Dockers.dependsOn Dockers.Type.Bullseye - (Some Network.Type.Berkeley) + network Profiles.Type.Standard Artifacts.Type.Rosetta } diff --git a/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall b/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall index b1229798a96..ebfea0150c1 100644 --- a/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall +++ b/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall @@ -26,6 +26,8 @@ let Artifacts = ../../Constants/Artifacts.dhall let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type +let network = Network.Type.Devnet + let dirtyWhen = [ S.strictlyStart (S.contains "src") , S.exactly "buildkite/src/Jobs/Test/RosettaIntegrationTests" "dhall" @@ -55,7 +57,8 @@ in Pipeline.build , Cmd.runInDocker Cmd.Docker::{ , image = - "gcr.io/o1labs-192920/mina-rosetta:\\\${MINA_DOCKER_TAG}-berkeley" + "gcr.io/o1labs-192920/mina-rosetta:\\\${MINA_DOCKER_TAG}-${Network.lowerName + network}" } "buildkite/scripts/rosetta-integration-tests-full.sh" ] @@ -66,7 +69,7 @@ in Pipeline.build , depends_on = Dockers.dependsOn Dockers.Type.Bullseye - (Some Network.Type.Berkeley) + network Profiles.Type.Standard Artifacts.Type.Rosetta } diff --git a/buildkite/src/Jobs/Test/RunSnarkProfiler.dhall b/buildkite/src/Jobs/Test/RunSnarkProfiler.dhall index 0523f1e1e74..accae521318 100644 --- a/buildkite/src/Jobs/Test/RunSnarkProfiler.dhall +++ b/buildkite/src/Jobs/Test/RunSnarkProfiler.dhall @@ -16,11 +16,14 @@ let RunInToolchain = ../../Command/RunInToolchain.dhall let Docker = ../../Command/Docker/Type.dhall +let Network = ../../Constants/Network.dhall + let Size = ../../Command/Size.dhall let dependsOn = DebianVersions.dependsOn DebianVersions.DebVersion.Bullseye + Network.Type.Devnet Profiles.Type.Standard let buildTestCmd diff --git a/buildkite/src/Jobs/Test/SingleNodeTest.dhall b/buildkite/src/Jobs/Test/SingleNodeTest.dhall index 74959dd325a..e5bbae08182 100644 --- a/buildkite/src/Jobs/Test/SingleNodeTest.dhall +++ b/buildkite/src/Jobs/Test/SingleNodeTest.dhall @@ -14,6 +14,8 @@ let DebianVersions = ../../Constants/DebianVersions.dhall let Profiles = ../../Constants/Profiles.dhall +let Network = ../../Constants/Network.dhall + let Docker = ../../Command/Docker/Type.dhall let Size = ../../Command/Size.dhall @@ -21,9 +23,11 @@ let Size = ../../Command/Size.dhall let dependsOn = DebianVersions.dependsOn DebianVersions.DebVersion.Bullseye + Network.Type.Devnet Profiles.Type.Lightnet # DebianVersions.dependsOn DebianVersions.DebVersion.Bullseye + Network.Type.Devnet Profiles.Type.Standard let buildTestCmd diff --git a/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall b/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall index f08d92218a8..545e4590570 100644 --- a/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall +++ b/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall @@ -21,12 +21,12 @@ let Artifacts = ../../Constants/Artifacts.dhall let dependsOn = Dockers.dependsOn Dockers.Type.Bullseye - (Some Network.Type.Devnet) + Network.Type.Devnet Profiles.Type.Standard Artifacts.Type.Rosetta # Dockers.dependsOn Dockers.Type.Bullseye - (None Network.Type) + Network.Type.Devnet Profiles.Type.Standard Artifacts.Type.Archive diff --git a/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall b/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall index d75e12b2a35..85a8b614ee2 100644 --- a/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall +++ b/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall @@ -21,12 +21,12 @@ let Artifacts = ../../Constants/Artifacts.dhall let dependsOn = Dockers.dependsOn Dockers.Type.Bullseye - (Some Network.Type.Devnet) + Network.Type.Devnet Profiles.Type.Standard Artifacts.Type.Daemon # Dockers.dependsOn Dockers.Type.Bullseye - (None Network.Type) + Network.Type.Devnet Profiles.Type.Standard Artifacts.Type.Archive diff --git a/buildkite/src/Jobs/Test/VersionLint.dhall b/buildkite/src/Jobs/Test/VersionLint.dhall index cb1301cc01d..83bdca8bbbb 100644 --- a/buildkite/src/Jobs/Test/VersionLint.dhall +++ b/buildkite/src/Jobs/Test/VersionLint.dhall @@ -16,7 +16,17 @@ let Docker = ../../Command/Docker/Type.dhall let Size = ../../Command/Size.dhall -let dependsOn = [ { name = "MinaArtifactBullseye", key = "build-deb-pkg" } ] +let DebianVersions = ../../Constants/DebianVersions.dhall + +let Network = ../../Constants/Network.dhall + +let Profiles = ../../Constants/Profiles.dhall + +let dependsOn = + DebianVersions.dependsOn + DebianVersions.DebVersion.Bullseye + Network.Type.Devnet + Profiles.Type.Standard let buildTestCmd : Text -> Size -> List Command.TaggedKey.Type -> Command.Type diff --git a/scripts/docker/helper.sh b/scripts/docker/helper.sh index 20b7bca95af..07cf02af46a 100644 --- a/scripts/docker/helper.sh +++ b/scripts/docker/helper.sh @@ -18,7 +18,7 @@ function export_base_image () { function export_version () { case "${SERVICE}" in - mina-daemon|mina-batch-txn) export VERSION="${VERSION}-${NETWORK##*=}" ;; + mina-daemon|mina-batch-txn|mina-rosetta) export VERSION="${VERSION}-${NETWORK##*=}" ;; *) ;; esac }