Skip to content

Commit

Permalink
fix prepare and monorepo
Browse files Browse the repository at this point in the history
  • Loading branch information
dkijania committed Sep 6, 2023
1 parent 0ffa950 commit 537b554
Show file tree
Hide file tree
Showing 2 changed files with 22 additions and 66 deletions.
78 changes: 15 additions & 63 deletions buildkite/src/Monorepo.dhall
Original file line number Diff line number Diff line change
Expand Up @@ -15,78 +15,31 @@ let PipelineStage = ./Pipeline/Stage.dhall
let Size = ./Command/Size.dhall
let triggerCommand = ./Pipeline/TriggerCommand.dhall

let mode = env:BUILDKITE_PIPELINE_MODE as Text ? "PullRequest"
let stage = env:BUILDKITE_PIPELINE_STAGE as Text ? "Stage1"

let jobs : List JobSpec.Type =
List/map
Pipeline.CompoundType
JobSpec.Type
(\(composite: Pipeline.CompoundType) -> composite.spec)
./gen/Jobs.dhall

let jobsByStage: List JobSpec.Type -> PipelineStage.Type -> List JobSpec.Type = \(jobs: List JobSpec.Type) -> \(stage: PipelineStage.Type) ->
Prelude.List.filter
JobSpec.Type
(\(job: JobSpec.Type) ->
PipelineStage.equal job.stage stage)
jobs

let jobsByMode: List JobSpec.Type -> PipelineMode.Type -> List JobSpec.Type = \(jobs: List JobSpec.Type) -> \(mode: PipelineMode.Type) ->
merge {
PullRequest = Prelude.List.filter
JobSpec.Type
(\(job: JobSpec.Type) ->
PipelineMode.equal job.mode mode)
jobs,
Stable = jobs
} mode

-- Run a job if we touched a dirty path
let makeCommand : JobSpec.Type -> Cmd.Type = \(job : JobSpec.Type) ->
let dirtyWhen = SelectFiles.compile job.dirtyWhen
let trigger = triggerCommand "src/Jobs/${job.path}/${job.name}.dhall"
let requestedPipelineName : Text = env:BUILDKITE_PIPELINE_MODE as Text? "PullRequest"
let jobPipelineName = PipelineMode.capitalName job.mode
let pipelineHandlers = {
PullRequest = ''
if [ "${requestedPipelineName}" == "PullRequest" ]; then
if (cat _computed_diff.txt | egrep -q '${dirtyWhen}'); then
echo "Triggering ${job.name} for reason:"
cat _computed_diff.txt | egrep '${dirtyWhen}'
${Cmd.format trigger}
fi
else
echo "Triggering ${job.name} because this is a stable buildkite run"
${Cmd.format trigger}
fi
'',
Stable = ''
if [ "${requestedPipelineName}" == "PullRequest" ]; then
echo "Skipping ${job.name} because this is a PR buildkite run"
else
echo "Triggering ${job.name} because this is a stable buildkite run"
${Cmd.format trigger}
fi
''
}
in Cmd.quietly (merge pipelineHandlers job.mode)

let stagedCommands: List JobSpec.Type -> PipelineStage.Type -> List Cmd.Type = \(jobs: List JobSpec.Type) -> \(stage: PipelineStage.Type) ->
Prelude.List.map JobSpec.Type Cmd.Type makeCommand (jobsByStage jobs stage)

let prefixCommands = [
Cmd.run "git config --global http.sslCAInfo /etc/ssl/certs/ca-bundle.crt", -- Tell git where to find certs for https connections
Cmd.run "git fetch origin", -- Freshen the cache
Cmd.run "./buildkite/scripts/generate-diff.sh > _computed_diff.txt"
]

let commands: PipelineStage.Type -> PipelineMode.Type -> List Cmd.Type = \(stage: PipelineStage.Type) -> \(mode: PipelineMode.Type) ->

-- Run a job if we touched a dirty path
let commands: Text -> Text -> List Cmd.Type = \(target_stage: Text) -> \(target_mode: Text) ->
Prelude.List.map
JobSpec.Type
Cmd.Type
(\(job: JobSpec.Type) ->
let job_mode = PipelineMode.lowerName job.mode
let job_stage = PipelineStage.lowerName job.stage
let target_mode = PipelineMode.lowerName mode
let target_stage = PipelineStage.lowerName stage
let job_mode = PipelineMode.capitalName job.mode
let job_stage = PipelineStage.capitalName job.stage

let dirtyWhen = SelectFiles.compile job.dirtyWhen
let trigger = triggerCommand "src/Jobs/${job.path}/${job.name}.dhall"
Expand All @@ -101,7 +54,7 @@ let commands: PipelineStage.Type -> PipelineMode.Type -> List Cmd.Type = \(sta
fi
'',
Stable = ''
if [ "${job_mode}" == "${target_mode}" ] && [ "${job_stage}" == "${target_stage}" ]; then
if [ "${job_stage}" == "${target_stage}" ]; then
echo "Triggering ${job.name} because this is a stable buildkite run"
${Cmd.format trigger}
fi
Expand All @@ -112,9 +65,8 @@ let commands: PipelineStage.Type -> PipelineMode.Type -> List Cmd.Type = \(sta
jobs

in
(
\(args : { stage : PipelineStage.Type, mode: PipelineMode.Type}) ->
Pipeline.build Pipeline.Config::{

let config : Pipeline.Config.Type = Pipeline.Config::{
spec = JobSpec::{
name = "monorepo-triage",
-- TODO: Clean up this code so we don't need an unused dirtyWhen here
Expand All @@ -123,9 +75,9 @@ in
steps = [
Command.build
Command.Config::{
commands = prefixCommands # (commands args.stage args.mode),
label = "Monorepo triage ${PipelineStage.capitalName args.stage}",
key = "cmds-${PipelineStage.lowerName args.stage}",
commands = prefixCommands # (commands stage mode),
label = "Monorepo triage ${stage}",
key = "cmds-${stage}",
target = Size.Small,
docker = Some Docker::{
image = (./Constants/ContainerImages.dhall).toolchainBase,
Expand All @@ -134,4 +86,4 @@ in
}
]
}
) : { stage : PipelineStage.Type, mode: PipelineMode.Type } -> Pipeline.CompoundType
in (Pipeline.build config).pipeline
10 changes: 7 additions & 3 deletions buildkite/src/Prepare.dhall
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,9 @@ let PipelineMode = ./Pipeline/Mode.dhall
let Size = ./Command/Size.dhall
let triggerCommand = ./Pipeline/TriggerCommand.dhall

let mode = env:BUILDKITE_PIPELINE_MODE as Text ? "PullRequest"
let stage = env:BUILDKITE_PIPELINE_STAGE as Text ? "Stage1"

let config : Pipeline.Config.Type = Pipeline.Config::{
spec = JobSpec::{
name = "prepare",
Expand All @@ -22,12 +25,13 @@ let config : Pipeline.Config.Type = Pipeline.Config::{
steps = [
Command.build Command.Config::{
commands = [
Cmd.run "export BUILDKITE_PIPELINE_MODE=${env:BUILDKITE_PIPELINE_MODE as Text ? "PullRequest"}",
Cmd.run "export BUILDKITE_PIPELINE_MODE=${mode}",
Cmd.run "export BUILDKITE_PIPELINE_STAGE=${stage}",
Cmd.run "./buildkite/scripts/generate-jobs.sh > buildkite/src/gen/Jobs.dhall",
triggerCommand "src/Monorepo.dhall"
],
label = "Prepare monorepo triage",
key = "monorepo-${args.stage}",
key = "monorepo-${stage}",
target = Size.Small,
docker = Some Docker::{
image = (./Constants/ContainerImages.dhall).toolchainBase,
Expand All @@ -36,4 +40,4 @@ let config : Pipeline.Config.Type = Pipeline.Config::{
}
]
}
in (Pipeline.build config).pipeline
in (Pipeline.build config).pipeline

0 comments on commit 537b554

Please sign in to comment.