gRPC Grafana dashboard

Signed-off-by: Alan Cha <Alan.cha1@ibm.com>
iter8-tools · Jul 27, 2023 · 7e633dd · 7e633dd
1 parent 70c9201
commit 7e633dd
Show file tree

Hide file tree

Showing 3 changed files with 1,047 additions and 211 deletions.
diff --git a/base/collect_grpc.go b/base/collect_grpc.go
@@ -2,6 +2,7 @@ package base
 
 import (
 	"fmt"
+	"os"
 	"time"
 
 	"github.com/bojand/ghz/runner"
@@ -38,6 +39,11 @@ type collectGRPCInputs struct {
 
 	// Endpoints is used to define multiple endpoints to test
 	Endpoints map[string]runner.Config `json:"endpoints" yaml:"endpoints"`
+
+	// Determines if Grafana dashboard should be created
+	// dasboard vs report/assess tasks
+	// TODO: remove
+	Grafana bool `json:"grafana" yaml:"grafana"`
 }
 
 // collectGRPCTask enables load testing of gRPC services.
@@ -49,6 +55,15 @@ type collectGRPCTask struct {
 	With collectGRPCInputs `json:"with" yaml:"with"`
 }
 
+// GHZResult is the raw data sent to the metrics server
+// This data will be transformed into httpDashboard when getGHZGrafana is called
+type GHZResult struct {
+	// key is the endpoint
+	EndpointResults map[string]runner.Report
+
+	Summary Insights
+}
+
 // initializeDefaults sets default values for the collect task
 func (t *collectGRPCTask) initializeDefaults() {
 	// set defaults
@@ -71,11 +86,11 @@ func (t *collectGRPCTask) validateInputs() error {
 }
 
 // resultForVersion collects gRPC test result for a given version
-func (t *collectGRPCTask) resultForVersion() (map[string]*runner.Report, error) {
+func (t *collectGRPCTask) resultForVersion() (map[string]runner.Report, error) {
 	// the main idea is to run ghz with proper options
 
 	var err error
-	results := map[string]*runner.Report{}
+	results := map[string]runner.Report{}
 
 	if len(t.With.Endpoints) > 0 {
 		log.Logger.Trace("multiple endpoints")
@@ -108,7 +123,11 @@ func (t *collectGRPCTask) resultForVersion() (map[string]*runner.Report, error)
 				continue
 			}
 
-			results[gRPCMetricPrefix+"-"+endpointID] = igr
+			resultsKey := gRPCMetricPrefix + "-" + endpointID
+			if t.With.Grafana {
+				resultsKey = endpoint.Call
+			}
+			results[resultsKey] = *igr
 		}
 	} else {
 		// TODO: supply all the allowed options
@@ -121,7 +140,11 @@ func (t *collectGRPCTask) resultForVersion() (map[string]*runner.Report, error)
 			return results, err
 		}
 
-		results[gRPCMetricPrefix] = igr
+		resultsKey := gRPCMetricPrefix
+		if t.With.Grafana {
+			resultsKey = t.With.Call
+		}
+		results[resultsKey] = *igr
 	}
 
 	return results, err
@@ -170,60 +193,80 @@ func (t *collectGRPCTask) run(exp *Experiment) error {
 	}
 	in := exp.Result.Insights
 
-	// 4. Populate all metrics collected by this task
-	for provider, data := range data {
-		// populate grpc request count
-		// todo: this logic breaks for looped experiments. Fix when we get to loops.
-		m := provider + "/" + gRPCRequestCountMetricName
-		mm := MetricMeta{
-			Description: "number of gRPC requests sent",
-			Type:        CounterMetricType,
-		}
-		if err = in.updateMetric(m, mm, 0, float64(data.Count)); err != nil {
-			return err
+	if t.With.Grafana {
+		// push data to metrics service
+		ghzResult := GHZResult{
+			EndpointResults: data,
+			Summary:         *exp.Result.Insights,
 		}
 
-		// populate error count & rate
-		ec := float64(0)
-		for _, count := range data.ErrorDist {
-			ec += float64(count)
+		// get URL of metrics server from environment variable
+		metricsServerURL, ok := os.LookupEnv(MetricsServerURL)
+		if !ok {
+			errorMessage := "could not look up METRICS_SERVER_URL environment variable"
+			log.Logger.Error(errorMessage)
+			return fmt.Errorf(errorMessage)
 		}
 
-		// populate count
-		// todo: This logic breaks for looped experiments. Fix when we get to loops.
-		m = provider + "/" + gRPCErrorCountMetricName
-		mm = MetricMeta{
-			Description: "number of responses that were errors",
-			Type:        CounterMetricType,
-		}
-		if err = in.updateMetric(m, mm, 0, ec); err != nil {
+		if err = putPerformanceResultToMetricsService(metricsServerURL, exp.Metadata.Namespace, exp.Metadata.Name, ghzResult); err != nil {
 			return err
 		}
+	} else {
+		// 4. Populate all metrics collected by this task
+		for provider, data := range data {
+			// populate grpc request count
+			// todo: this logic breaks for looped experiments. Fix when we get to loops.
+			m := provider + "/" + gRPCRequestCountMetricName
+			mm := MetricMeta{
+				Description: "number of gRPC requests sent",
+				Type:        CounterMetricType,
+			}
+			if err = in.updateMetric(m, mm, 0, float64(data.Count)); err != nil {
+				return err
+			}
+
+			// populate error count & rate
+			ec := float64(0)
+			for _, count := range data.ErrorDist {
+				ec += float64(count)
+			}
 
-		// populate rate
-		// todo: This logic breaks for looped experiments. Fix when we get to loops.
-		m = provider + "/" + gRPCErrorRateMetricName
-		rc := float64(data.Count)
-		if rc != 0 {
+			// populate count
+			// todo: This logic breaks for looped experiments. Fix when we get to loops.
+			m = provider + "/" + gRPCErrorCountMetricName
 			mm = MetricMeta{
-				Description: "fraction of responses that were errors",
-				Type:        GaugeMetricType,
+				Description: "number of responses that were errors",
+				Type:        CounterMetricType,
 			}
-			if err = in.updateMetric(m, mm, 0, ec/rc); err != nil {
+			if err = in.updateMetric(m, mm, 0, ec); err != nil {
 				return err
 			}
-		}
 
-		// populate latency sample
-		m = provider + "/" + gRPCLatencySampleMetricName
-		mm = MetricMeta{
-			Description: "gRPC Latency Sample",
-			Type:        SampleMetricType,
-			Units:       StringPointer("msec"),
-		}
-		lh := latencySample(data.Details)
-		if err = in.updateMetric(m, mm, 0, lh); err != nil {
-			return err
+			// populate rate
+			// todo: This logic breaks for looped experiments. Fix when we get to loops.
+			m = provider + "/" + gRPCErrorRateMetricName
+			rc := float64(data.Count)
+			if rc != 0 {
+				mm = MetricMeta{
+					Description: "fraction of responses that were errors",
+					Type:        GaugeMetricType,
+				}
+				if err = in.updateMetric(m, mm, 0, ec/rc); err != nil {
+					return err
+				}
+			}
+
+			// populate latency sample
+			m = provider + "/" + gRPCLatencySampleMetricName
+			mm = MetricMeta{
+				Description: "gRPC Latency Sample",
+				Type:        SampleMetricType,
+				Units:       StringPointer("msec"),
+			}
+			lh := latencySample(data.Details)
+			if err = in.updateMetric(m, mm, 0, lh); err != nil {
+				return err
+			}
 		}
 	}