*: init multirunners server command

Dockerfile

@@ -9,12 +9,12 @@ RUN mkdir -p /output
 WORKDIR /kperf-build
 RUN --mount=source=./,target=/kperf-build,rw make build && PREFIX=/output make install
 
-FROM gcr.io/distroless/static-debian12:nonroot AS release-stage
+# TODO: We should consider to implement our own curl to upload data
+FROM ubuntu:22.04 AS release-stage
+
+RUN apt update -y && apt install curl -y
 
 WORKDIR /
 
 COPY --from=build-stage /output/bin/kperf /kperf
-
-USER nonroot:nonroot
-
-ENTRYPOINT ["/kperf"]
+COPY scripts/run_runner.sh /run_runner.sh

api/types/http.go

@@ -0,0 +1,6 @@
+package types
+
+// HTTPError is used to render response for error.
+type HTTPError struct {
+	Error string `json:"error"`
+}

api/types/runner_group.go

@@ -0,0 +1,22 @@
+package types
+
+// RunnerGroup defines a set of runners with same load profile.
+type RunnerGroup struct {
+	// Name is the name of runner group.
+	Name string `json:"name" yaml:"name"`
+	// Spec is specification of the desired behavior of the runner group.
+	Spec *RunnerGroupSpec `json:"spec" yaml:"spec"`
+}
+
+// RunnerGroupSpec is to descibe how the runner group works.
+type RunnerGroupSpec struct {
+	// Count is the number of runners.
+	Count int32 `json:"count" yaml:"count"`
+	// Profile defines what the load traffic looks like.
+	Profile *LoadProfile `json:"loadProfile,omitempty" yaml:"loadProfile"`
+	// NodeAffinity defines how to deploy runners into dedicated nodes
+	// which have specific labels.
+	NodeAffinity map[string][]string `json:"nodeAffinity,omitempty" yaml:"nodeAffinity"`
+	// ServiceAccount is the name of the ServiceAccount to use to run runners.
+	ServiceAccount *string `json:"serviceAccount,omitempty" yaml:"serviceAccount"`
+}

cmd/kperf/commands/multirunners/runner.go

@@ -71,10 +71,17 @@ var Command = cli.Command{
 	Name:      "multirunners",
 	ShortName: "mrunners",
 	Usage:     "packages runner as job and deploy runners into kubernetes",
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:  "kubeconfig",
+			Usage: "Path to the kubeconfig file",
+		},
+	},
 	Subcommands: []cli.Command{
 		runCommand,
 		waitCommand,
 		resultCommand,
+		serverCommand,
 	},
 }
 

cmd/kperf/commands/multirunners/server.go

@@ -0,0 +1,109 @@
+package multirunners
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/Azure/kperf/runner"
+	runnergroup "github.com/Azure/kperf/runner/group"
+
+	"github.com/urfave/cli"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+)
+
+var serverCommand = cli.Command{
+	Name: "server",
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:  "namespace",
+			Usage: "The namespace scope for runners",
+			Value: "default",
+		},
+		cli.StringSliceFlag{
+			Name:     "runners",
+			Usage:    "The runner spec's URI",
+			Required: true,
+		},
+		cli.StringFlag{
+			Name:     "runner-image",
+			Usage:    "The runner's conainer image",
+			Required: true,
+		},
+		cli.StringSliceFlag{
+			Name:     "address",
+			Usage:    "Address for the server",
+			Required: true,
+		},
+		cli.StringFlag{
+			Name:     "data",
+			Usage:    "The runner result should be stored in that path",
+			Required: true,
+		},
+	},
+	Hidden: true,
+	Action: func(cliCtx *cli.Context) error {
+		if cliCtx.NArg() != 1 {
+			return fmt.Errorf("required only one argument as server name")
+		}
+
+		name := strings.TrimSpace(cliCtx.Args().Get(0))
+		if len(name) == 0 {
+			return fmt.Errorf("required non-empty server name")
+		}
+
+		groupHandlers, err := buildRunnerGroupHandlers(cliCtx, name)
+		if err != nil {
+			return fmt.Errorf("failed to create runner group handlers: %w", err)
+		}
+
+		dataDir := cliCtx.String("data")
+		addrs := cliCtx.StringSlice("address")
+
+		srv, err := runner.NewServer(dataDir, addrs, groupHandlers...)
+		if err != nil {
+			return err
+		}
+		return srv.Run()
+	},
+}
+
+// buildRunnerGroupHandlers creates a slice of runner group handlers.
+func buildRunnerGroupHandlers(cliCtx *cli.Context, serverName string) ([]*runnergroup.Handler, error) {
+	clientset, err := buildKubernetesClientset(cliCtx)
+	if err != nil {
+		return nil, fmt.Errorf("failed to build kubernetes clientset: %w", err)
+	}
+
+	specURIs := cliCtx.StringSlice("runners")
+	imgRef := cliCtx.String("runner-image")
+	namespace := cliCtx.String("namespace")
+
+	groups := make([]*runnergroup.Handler, 0, len(specURIs))
+	for idx, specURI := range specURIs {
+		spec, err := runnergroup.NewRunnerGroupSpecFromURI(clientset, specURI)
+		if err != nil {
+			return nil, err
+		}
+
+		groupName := fmt.Sprintf("%s-%d", serverName, idx)
+		g, err := runnergroup.NewHandler(clientset, namespace, groupName, spec, imgRef)
+		if err != nil {
+			return nil, err
+		}
+		groups = append(groups, g)
+	}
+
+	return groups, nil
+}
+
+// buildKubernetesClientset builds kubernetes clientset from global flag.
+func buildKubernetesClientset(cliCtx *cli.Context) (kubernetes.Interface, error) {
+	kubeCfgPath := cliCtx.GlobalString("kubeconfig")
+	config, err := clientcmd.BuildConfigFromFlags("", kubeCfgPath)
+	if err != nil {
+		return nil, err
+	}
+
+	return kubernetes.NewForConfig(config)
+}

go.mod

@@ -3,16 +3,19 @@ module github.com/Azure/kperf
 go 1.20
 
 require (
+	github.com/gorilla/mux v1.8.0
 	github.com/stretchr/testify v1.8.4
 	github.com/urfave/cli v1.22.14
 	golang.org/x/time v0.3.0
 	gopkg.in/yaml.v2 v2.4.0
 	helm.sh/helm/v3 v3.13.3
+	k8s.io/api v0.28.4
 	k8s.io/apimachinery v0.28.4
 	k8s.io/cli-runtime v0.28.4
 	k8s.io/client-go v0.28.4
 	k8s.io/klog/v2 v2.100.1
 	k8s.io/kubectl v0.28.4
+	k8s.io/utils v0.0.0-20230406110748-d93618cff8a2
 )
 
 require (
@@ -62,7 +65,6 @@ require (
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/google/uuid v1.3.0 // indirect
-	github.com/gorilla/mux v1.8.0 // indirect
 	github.com/gosuri/uitable v0.0.4 // indirect
 	github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
@@ -132,12 +134,10 @@ require (
 	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/api v0.28.4 // indirect
 	k8s.io/apiextensions-apiserver v0.28.4 // indirect
 	k8s.io/apiserver v0.28.4 // indirect
 	k8s.io/component-base v0.28.4 // indirect
 	k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect
-	k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect
 	oras.land/oras-go v1.2.4 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 // indirect