From a718eef0dfaaa0bc9134a77bdde8c896ba320090 Mon Sep 17 00:00:00 2001 From: Jaskaran Sarkaria Date: Thu, 31 Oct 2024 08:15:28 +0000 Subject: [PATCH] =?UTF-8?q?perf:=20=E2=9A=A1=EF=B8=8F=20gradually=20increa?= =?UTF-8?q?se=20apply=20concurrency=20(#653)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/environment/apply.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/environment/apply.go b/pkg/environment/apply.go index ff2f2355..0e212c06 100644 --- a/pkg/environment/apply.go +++ b/pkg/environment/apply.go @@ -189,7 +189,7 @@ func (a *Apply) applyNamespaceDirs(chunkFolder []string) error { chunkStream := util.Generator(done, chunkFolder...) - routineResults := a.parallelApplyNamespace(done, chunkStream, 1) // goroutines are very lightweight and can number in millions, but the tasks we are doing are very heavy so we need to limit this as much as possible + routineResults := a.parallelApplyNamespace(done, chunkStream, 2) // goroutines are very lightweight and can number in millions, but the tasks we are doing are very heavy so we need to limit this as much as possible results := util.FanIn(done, routineResults...) @@ -203,7 +203,7 @@ func (a *Apply) applyNamespaceDirs(chunkFolder []string) error { func (a *Apply) parallelApplyNamespace(done <-chan bool, dirStream <-chan string, numRoutines int) []<-chan string { if a.Options.IsApplyPipeline { - runtime.GOMAXPROCS(1) // this is based on https://github.com/ministryofjustice/cloud-platform-infrastructure/blob/ebafd84ba45a18deeb113d1b57f565141368c187/terraform/aws-accounts/cloud-platform-aws/vpc/eks/cluster.tf#L46C1-L46C58 current max cpu is 4 (for workloads running in concourse) + runtime.GOMAXPROCS(numRoutines) // this is based on https://github.com/ministryofjustice/cloud-platform-infrastructure/blob/ebafd84ba45a18deeb113d1b57f565141368c187/terraform/aws-accounts/cloud-platform-aws/vpc/eks/cluster.tf#L46C1-L46C58 current max cpu is 4 (for workloads running in concourse) } routineResults := make([]<-chan string, numRoutines)