Added failure list code, tested

api/types/metric.go

@@ -6,14 +6,14 @@ import "time"
 type ResponseStats struct {
 	// Total represents total number of requests.
 	Total int
-	// Failures represents number of failure request.
-	Failures int
+	// List of failures
+	FailureList []error
 	// Duration means the time of benchmark.
 	Duration time.Duration
-	// Latencies represents the latency distribution in seconds.
+	// PercentileLatencies represents the latency distribution in seconds.
 	//
 	// NOTE: The key represents quantile.
-	Latencies map[float64]float64
+	PercentileLatencies map[float64]float64
 	// TODO:
 	// 1. Support total read/upload bytes
 	// 2. Support failures partitioned by http code and verb

cmd/kperf/commands/root.go

@@ -3,6 +3,7 @@ package commands
 import (
 	"github.com/Azure/kperf/cmd/kperf/commands/multirunners"
 	"github.com/Azure/kperf/cmd/kperf/commands/runner"
+	"github.com/Azure/kperf/cmd/kperf/commands/virtualcluster"
 
 	"github.com/urfave/cli"
 )
@@ -15,6 +16,7 @@ func App() *cli.App {
 		Commands: []cli.Command{
 			runner.Command,
 			multirunners.Command,
+			virtualcluster.Command,
 		},
 	}
 }

cmd/kperf/commands/runner/runner.go

@@ -95,11 +95,15 @@ func loadConfig(cliCtx *cli.Context) (*types.LoadProfile, error) {
 	}
 
 	// override value by flags
-	//
-	// TODO(weifu): do not override if flag is not set
-	profileCfg.Spec.Rate = cliCtx.Int("rate")
-	profileCfg.Spec.Conns = cliCtx.Int("conns")
-	profileCfg.Spec.Total = cliCtx.Int("total")
+	if v := "rate"; cliCtx.IsSet(v) {
+		profileCfg.Spec.Rate = cliCtx.Int(v)
+	}
+	if v := "conns"; cliCtx.IsSet(v) || profileCfg.Spec.Conns == 0 {
+		profileCfg.Spec.Conns = cliCtx.Int(v)
+	}
+	if v := "total"; cliCtx.IsSet(v) || profileCfg.Spec.Total == 0 {
+		profileCfg.Spec.Total = cliCtx.Int(v)
+	}
 
 	if err := profileCfg.Validate(); err != nil {
 		return nil, err
@@ -111,18 +115,23 @@ func loadConfig(cliCtx *cli.Context) (*types.LoadProfile, error) {
 func printResponseStats(stats *types.ResponseStats) {
 	fmt.Println("Response stat:")
 	fmt.Printf("  Total: %v\n", stats.Total)
-	fmt.Printf("  Failures: %v\n", stats.Failures)
+	fmt.Printf(" Total Failures: %v\n", len(stats.FailureList))
+	fmt.Println("==========================")
+	for i, v := range stats.FailureList {
+		fmt.Printf("  Failure %d: %v\n", i+1, v)
+	}
+	fmt.Println("==========================")
 	fmt.Printf("  Duration: %v\n", stats.Duration)
 	fmt.Printf("  Requests/sec: %.2f\n", float64(stats.Total)/stats.Duration.Seconds())
 
 	fmt.Println("  Latency Distribution:")
-	keys := make([]float64, 0, len(stats.Latencies))
-	for q := range stats.Latencies {
+	keys := make([]float64, 0, len(stats.PercentileLatencies))
+	for q := range stats.PercentileLatencies {
 		keys = append(keys, q)
 	}
 	sort.Float64s(keys)
 
 	for _, q := range keys {
-		fmt.Printf("    [%.2f] %.3fs\n", q, stats.Latencies[q])
+		fmt.Printf("    [%.2f] %.3fs\n", q/100.0, stats.PercentileLatencies[q])
 	}
 }

cmd/kperf/commands/virtualcluster/nodepool.go

@@ -0,0 +1,67 @@
+package virtualcluster
+
+import (
+	"fmt"
+
+	"github.com/urfave/cli"
+)
+
+var nodepoolCommand = cli.Command{
+	Name:  "nodepool",
+	Usage: "Manage virtual node pools",
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:  "kubeconfig",
+			Usage: "Path to the kubeconfig file",
+		},
+	},
+	Subcommands: []cli.Command{
+		nodepoolAddCommand,
+		nodepoolDelCommand,
+		nodepoolListCommand,
+	},
+}
+
+var nodepoolAddCommand = cli.Command{
+	Name:      "add",
+	Usage:     "Add a virtual node pool",
+	ArgsUsage: "NAME",
+	Flags: []cli.Flag{
+		cli.IntFlag{
+			Name:  "nodes",
+			Usage: "The number of virtual nodes",
+			Value: 10,
+		},
+		cli.IntFlag{
+			Name:  "cpu",
+			Usage: "The allocatable CPU resource per node",
+			Value: 8,
+		},
+		cli.IntFlag{
+			Name:  "memory",
+			Usage: "The allocatable Memory resource per node (GiB)",
+			Value: 16,
+		},
+	},
+	Action: func(cliCtx *cli.Context) error {
+		return fmt.Errorf("nodepool add - not implemented")
+	},
+}
+
+var nodepoolDelCommand = cli.Command{
+	Name:      "delete",
+	ShortName: "del",
+	ArgsUsage: "NAME",
+	Usage:     "Delete a virtual node pool",
+	Action: func(cliCtx *cli.Context) error {
+		return fmt.Errorf("nodepool delete - not implemented")
+	},
+}
+
+var nodepoolListCommand = cli.Command{
+	Name:  "list",
+	Usage: "List virtual node pools",
+	Action: func(cliCtx *cli.Context) error {
+		return fmt.Errorf("nodepool list - not implemented")
+	},
+}

cmd/kperf/commands/virtualcluster/vc.go

@@ -0,0 +1,15 @@
+package virtualcluster
+
+import "github.com/urfave/cli"
+
+// const namespace = "kperf-virtualcluster"
+
+// Command represents virtualcluster subcommand.
+var Command = cli.Command{
+	Name:      "virtualcluster",
+	ShortName: "vc",
+	Usage:     "Setup virtual cluster and run workload on that",
+	Subcommands: []cli.Command{
+		nodepoolCommand,
+	},
+}

go.mod

@@ -3,7 +3,6 @@ module github.com/Azure/kperf
 go 1.20
 
 require (
-	github.com/prometheus/client_golang v1.17.0
 	github.com/stretchr/testify v1.8.4
 	github.com/urfave/cli v1.22.14
 	golang.org/x/time v0.3.0
@@ -14,8 +13,6 @@ require (
 )
 
 require (
-	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/go-logr/logr v1.2.4 // indirect
@@ -24,13 +21,9 @@ require (
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/imdario/mergo v0.3.6 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect
-	github.com/prometheus/common v0.44.0 // indirect
-	github.com/prometheus/procfs v0.11.1 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	golang.org/x/net v0.17.0 // indirect

go.sum

@@ -1,8 +1,4 @@
 github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
-github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
-github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
-github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
-github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -17,7 +13,6 @@ github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2Kv
 github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
-github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
@@ -39,8 +34,6 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
-github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
-github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -49,14 +42,6 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q=
-github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY=
-github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLqVdK4VrYkVD5diGdl4sxJurKJEMnODWRJlxV9oM=
-github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU=
-github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY=
-github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY=
-github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI=
-github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
@@ -88,7 +73,6 @@ golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
 golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8=
 golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE=
-golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

metrics/request.go

@@ -1,64 +1,80 @@
 package metrics
 
 import (
-	"fmt"
-	"sync/atomic"
-
-	"github.com/prometheus/client_golang/prometheus"
+	"container/list"
+	"math"
+	"sort"
+	"sync"
 )
 
 // ResponseMetric is a measurement related to http response.
 type ResponseMetric interface {
 	// ObserveLatency observes latency.
 	ObserveLatency(seconds float64)
 	// ObserveFailure observes failure response.
-	ObserveFailure()
+	ObserveFailure(err error)
 	// Gather returns the summary.
-	Gather() (latencies map[float64]float64, failure int, _ error)
+	Gather() (latencies []float64, percentileLatencies map[float64]float64, failureList []error)
 }
 
 type responseMetricImpl struct {
-	latencySeconds *prometheus.SummaryVec
-	failureCount   int64
+	mu          sync.Mutex
+	failureList []error
+	latencies   *list.List
 }
 
 func NewResponseMetric() ResponseMetric {
 	return &responseMetricImpl{
-		latencySeconds: prometheus.NewSummaryVec(
-			prometheus.SummaryOpts{
-				Namespace:  "request",
-				Name:       "request_latency_seconds",
-				Objectives: map[float64]float64{0: 0, 0.5: 0, 0.9: 0, 0.95: 0, 0.99: 0, 1: 0},
-			},
-			[]string{},
-		),
+		latencies:   list.New(),
+		failureList: []error{},
 	}
 }
 
 // ObserveLatency implements ResponseMetric.
 func (m *responseMetricImpl) ObserveLatency(seconds float64) {
-	m.latencySeconds.WithLabelValues().Observe(seconds)
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.latencies.PushBack(seconds)
 }
 
 // ObserveFailure implements ResponseMetric.
-func (m *responseMetricImpl) ObserveFailure() {
-	atomic.AddInt64(&m.failureCount, 1)
+func (m *responseMetricImpl) ObserveFailure(err error) {
+	m.failureList = append(m.failureList, err)
 }
 
 // Gather implements ResponseMetric.
-func (m *responseMetricImpl) Gather() (map[float64]float64, int, error) {
-	reg := prometheus.NewRegistry()
-	reg.MustRegister(m.latencySeconds)
+func (m *responseMetricImpl) Gather() ([]float64, map[float64]float64, []error) {
+	latencies := m.dumpLatencies()
+	return latencies, buildPercentileLatencies(latencies), m.failureList
+}
 
-	metricFamilies, err := reg.Gather()
-	if err != nil {
-		return nil, 0, fmt.Errorf("failed to gather from local registry: %w", err)
+func (m *responseMetricImpl) dumpLatencies() []float64 {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	res := make([]float64, 0, m.latencies.Len())
+	for e := m.latencies.Front(); e != nil; e = e.Next() {
+		res = append(res, e.Value.(float64))
 	}
+	return res
+}
 
-	latencies := map[float64]float64{}
-	for _, q := range metricFamilies[0].GetMetric()[0].GetSummary().GetQuantile() {
-		latencies[q.GetQuantile()] = q.GetValue()
+var percentiles = []float64{0, 50, 90, 95, 99, 100}
+
+func buildPercentileLatencies(latencies []float64) map[float64]float64 {
+	if len(latencies) == 0 {
+		return nil
 	}
 
-	return latencies, int(atomic.LoadInt64(&m.failureCount)), nil
+	res := make(map[float64]float64, len(percentiles))
+
+	n := len(latencies)
+	sort.Float64s(latencies)
+	for _, p := range percentiles {
+		idx := int(math.Ceil(float64(n) * p / 100))
+		if idx > 0 {
+			idx--
+		}
+		res[p] = latencies[idx]
+	}
+	return res
 }