From 66da903d55c3bc3d98fdb32bc551746283b99c46 Mon Sep 17 00:00:00 2001 From: Wei Fu Date: Wed, 30 Oct 2024 23:12:36 +0000 Subject: [PATCH] *: use warmup as a general-purpose name Signed-off-by: Wei Fu --- contrib/cmd/runkperf/commands/root.go | 4 +- .../commands/{ekswarmup => warmup}/command.go | 70 ++++++++++++++----- .../{ekswarmup.yaml => warmup.yaml} | 3 - contrib/internal/utils/utils.go | 6 -- 4 files changed, 54 insertions(+), 29 deletions(-) rename contrib/cmd/runkperf/commands/{ekswarmup => warmup}/command.go (72%) rename contrib/internal/manifests/loadprofile/{ekswarmup.yaml => warmup.yaml} (90%) diff --git a/contrib/cmd/runkperf/commands/root.go b/contrib/cmd/runkperf/commands/root.go index a30f495..4a32fd1 100644 --- a/contrib/cmd/runkperf/commands/root.go +++ b/contrib/cmd/runkperf/commands/root.go @@ -10,7 +10,7 @@ import ( "strconv" "github.com/Azure/kperf/contrib/cmd/runkperf/commands/bench" - "github.com/Azure/kperf/contrib/cmd/runkperf/commands/ekswarmup" + "github.com/Azure/kperf/contrib/cmd/runkperf/commands/warmup" "github.com/urfave/cli" "k8s.io/klog/v2" @@ -22,7 +22,7 @@ func App() *cli.App { Name: "runkperf", // TODO: add more fields Commands: []cli.Command{ - ekswarmup.Command, + warmup.Command, bench.Command, }, Flags: []cli.Flag{ diff --git a/contrib/cmd/runkperf/commands/ekswarmup/command.go b/contrib/cmd/runkperf/commands/warmup/command.go similarity index 72% rename from contrib/cmd/runkperf/commands/ekswarmup/command.go rename to contrib/cmd/runkperf/commands/warmup/command.go index bdc1ba4..ae5937f 100644 --- a/contrib/cmd/runkperf/commands/ekswarmup/command.go +++ b/contrib/cmd/runkperf/commands/warmup/command.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package ekswarmup +package warmup import ( "context" @@ -10,6 +10,7 @@ import ( "time" "github.com/Azure/kperf/api/types" + kperfcmdutils "github.com/Azure/kperf/cmd/kperf/commands/utils" "github.com/Azure/kperf/contrib/internal/utils" "github.com/urfave/cli" @@ -21,10 +22,10 @@ import ( "k8s.io/klog/v2" ) -// Command represents ekswarmup subcommand. +// Command represents warmup subcommand. var Command = cli.Command{ - Name: "ekswarmup", - Usage: "Warmup EKS cluster and try best to scale it to 8 cores at least", + Name: "warmup", + Usage: "Warmup cluster and try best to scale it to 8 cores at least", Flags: []cli.Flag{ cli.StringFlag{ Name: "kubeconfig", @@ -55,12 +56,27 @@ var Command = cli.Command{ Usage: "Total requests per runner (There are 10 runners totally and runner's rate is 20)", Value: 10000, }, + cli.StringFlag{ + Name: "vc-affinity", + Usage: "Deploy virtualnode's controller with a specific labels (FORMAT: KEY=VALUE[,VALUE])", + Value: "node.kubernetes.io/instance-type=Standard_D8s_v3,m4.2xlarge", + }, + cli.StringFlag{ + Name: "rg-affinity", + Usage: "Deploy runner group with a specific labels (FORMAT: KEY=VALUE[,VALUE])", + Value: "node.kubernetes.io/instance-type=Standard_D16s_v3,m4.4xlarge", + }, + cli.BoolFlag{ + Name: "eks", + Usage: "Indicates the target kubernetes cluster is EKS", + Hidden: true, + }, }, Action: func(cliCtx *cli.Context) (retErr error) { ctx := context.Background() rgCfgFile, rgCfgFileDone, err := utils.NewLoadProfileFromEmbed( - "loadprofile/ekswarmup.yaml", + "loadprofile/warmup.yaml", func(spec *types.RunnerGroupSpec) error { reqs := cliCtx.Int("total") if reqs < 0 { @@ -72,8 +88,15 @@ var Command = cli.Command{ return fmt.Errorf("invalid rate value: %v", rate) } + rgAffinity := cliCtx.String("rg-affinity") + affinityLabels, err := kperfcmdutils.KeyValuesMap([]string{rgAffinity}) + if err != nil { + return fmt.Errorf("failed to parse %s affinity: %w", rgAffinity, err) + } + spec.Profile.Spec.Total = reqs spec.Profile.Spec.Rate = rate + spec.NodeAffinity = affinityLabels data, _ := yaml.Marshal(spec) klog.V(2).InfoS("Load Profile", "config", string(data)) @@ -86,10 +109,14 @@ var Command = cli.Command{ defer func() { _ = rgCfgFileDone() }() kubeCfgPath := cliCtx.String("kubeconfig") + isEKS := cliCtx.Bool("eks") + virtualNodeAffinity := cliCtx.String("vc-affinity") - perr := patchEKSDaemonsetWithoutToleration(ctx, kubeCfgPath) - if perr != nil { - return perr + if isEKS { + perr := patchEKSDaemonsetWithoutToleration(ctx, kubeCfgPath) + if perr != nil { + return perr + } } cores, ferr := utils.FetchAPIServerCores(ctx, kubeCfgPath) @@ -102,7 +129,7 @@ var Command = cli.Command{ klog.V(0).ErrorS(ferr, "failed to fetch apiserver cores") } - delNP, err := deployWarmupVirtualNodepool(ctx, kubeCfgPath) + delNP, err := deployWarmupVirtualNodepool(ctx, kubeCfgPath, isEKS, virtualNodeAffinity) if err != nil { return err } @@ -155,15 +182,22 @@ func isReady(cores map[string]int) bool { return n >= 2 } -// deployWarmupVirtualNodepool deploys nodepool on m4.2xlarge nodes for warmup. -func deployWarmupVirtualNodepool(ctx context.Context, kubeCfgPath string) (func() error, error) { +// deployWarmupVirtualNodepool deploys virtual nodepool. +func deployWarmupVirtualNodepool(ctx context.Context, kubeCfgPath string, isEKS bool, nodeAffinity string) (func() error, error) { target := "warmup" - kr := utils.NewKperfRunner(kubeCfgPath, "") klog.V(0).InfoS("Deploying virtual nodepool", "name", target) - sharedProviderID, err := utils.FetchNodeProviderIDByType(ctx, kubeCfgPath, utils.EKSIdleNodepoolInstanceType) - if err != nil { - return nil, fmt.Errorf("failed to get placeholder providerID: %w", err) + + kr := utils.NewKperfRunner(kubeCfgPath, "") + + sharedProviderID := "" + var err error + + if isEKS { + sharedProviderID, err = utils.FetchNodeProviderIDByType(ctx, kubeCfgPath, utils.EKSIdleNodepoolInstanceType) + if err != nil { + return nil, fmt.Errorf("failed to get placeholder providerID: %w", err) + } } klog.V(0).InfoS("Trying to delete", "nodepool", target) @@ -171,8 +205,7 @@ func deployWarmupVirtualNodepool(ctx context.Context, kubeCfgPath string) (func( klog.V(0).ErrorS(err, "failed to delete", "nodepool", target) } - err = kr.NewNodepool(ctx, 0, target, 100, 32, 96, 110, - "node.kubernetes.io/instance-type=m4.2xlarge", sharedProviderID) + err = kr.NewNodepool(ctx, 0, target, 100, 32, 96, 110, nodeAffinity, sharedProviderID) if err != nil { return nil, fmt.Errorf("failed to create nodepool %s: %w", target, err) } @@ -185,8 +218,9 @@ func deployWarmupVirtualNodepool(ctx context.Context, kubeCfgPath string) (func( // patchEKSDaemonsetWithoutToleration removes tolerations to avoid pod scheduled // to virtual nodes. func patchEKSDaemonsetWithoutToleration(ctx context.Context, kubeCfgPath string) error { - clientset := mustClientset(kubeCfgPath) + klog.V(0).Info("Trying to removes EKS Daemonset's tolerations to avoid pod scheduled to virtual nodes") + clientset := mustClientset(kubeCfgPath) ds := clientset.AppsV1().DaemonSets("kube-system") for _, dn := range []string{"aws-node", "kube-proxy"} { d, err := ds.Get(ctx, dn, metav1.GetOptions{}) diff --git a/contrib/internal/manifests/loadprofile/ekswarmup.yaml b/contrib/internal/manifests/loadprofile/warmup.yaml similarity index 90% rename from contrib/internal/manifests/loadprofile/ekswarmup.yaml rename to contrib/internal/manifests/loadprofile/warmup.yaml index a2573eb..85e60f8 100644 --- a/contrib/internal/manifests/loadprofile/ekswarmup.yaml +++ b/contrib/internal/manifests/loadprofile/warmup.yaml @@ -25,6 +25,3 @@ loadProfile: resource: events limit: 1000 shares: 100 # chance 100 / (1000 + 100 + 100) -nodeAffinity: - node.kubernetes.io/instance-type: - - m4.4xlarge diff --git a/contrib/internal/utils/utils.go b/contrib/internal/utils/utils.go index e4c0f62..ccfddd5 100644 --- a/contrib/internal/utils/utils.go +++ b/contrib/internal/utils/utils.go @@ -39,12 +39,6 @@ var ( // provider ID for all the virtual nodes so that EKS cloud provider // won't delete our virtual nodes. EKSIdleNodepoolInstanceType = "m4.large" - - // EKSRunnerNodepoolInstanceType is the instance type of nodes for kperf - // runners. - // - // NOTE: This is default type. Please align it with ../manifests/loadprofile/ekswarmup.yaml. - EKSRunnerNodepoolInstanceType = "m4.4xlarge" ) // RepeatJobWithPod repeats to deploy 3k pods.