Add a Thresholds.Validate method

The newly introduced Validate method ensures that for a given
collection of Thresholds, they apply to existing metrics/submetrics,
and apply methods that are supported by the former.

metrics/metric_type.go

@@ -84,3 +84,39 @@ func (t MetricType) String() string {
 		return "[INVALID]"
 	}
 }
+
+// supportedAggregationMethods returns the list of threshold aggregation methods
+// that can be used against this MetricType.
+func (t MetricType) supportedAggregationMethods() []string {
+	switch t {
+	case Counter:
+		return []string{tokenCount, tokenRate}
+	case Gauge:
+		return []string{tokenValue}
+	case Rate:
+		return []string{tokenRate}
+	case Trend:
+		return []string{
+			tokenAvg,
+			tokenMin,
+			tokenMax,
+			tokenMed,
+			tokenPercentile,
+		}
+	default:
+		// unreachable!
+		return nil
+	}
+}
+
+// supportsAggregationMethod returns whether the MetricType supports a
+// given threshold aggregation method or not.
+func (t MetricType) supportsAggregationMethod(aggregationMethod string) bool {
+	for _, m := range t.supportedAggregationMethods() {
+		if aggregationMethod == m {
+			return true
+		}
+	}
+
+	return false
+}

metrics/thresholds.go

@@ -22,10 +22,13 @@ package metrics
 import (
 	"bytes"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"strings"
 	"time"
 
+	"go.k6.io/k6/errext"
+	"go.k6.io/k6/errext/exitcodes"
 	"go.k6.io/k6/lib/types"
 )
 
@@ -56,7 +59,7 @@ func newThreshold(src string, abortOnFail bool, gracePeriod types.NullDuration)
 func (t *Threshold) runNoTaint(sinks map[string]float64) (bool, error) {
 	// Extract the sink value for the aggregation method used in the threshold
 	// expression
-	lhs, ok := sinks[t.parsed.AggregationMethod]
+	lhs, ok := sinks[t.parsed.SinkKey()]
 	if !ok {
 		return false, fmt.Errorf("unable to apply threshold %s over metrics; reason: "+
 			"no metric supporting the %s aggregation method found",
@@ -210,11 +213,12 @@ func (ts *Thresholds) Run(sink Sink, duration time.Duration) (bool, error) {
 		// Parse the percentile thresholds and insert them in
 		// the sinks mapping.
 		for _, threshold := range ts.Thresholds {
-			if !strings.HasPrefix(threshold.parsed.AggregationMethod, "p(") {
+			if threshold.parsed.AggregationMethod != tokenPercentile {
 				continue
 			}
 
-			ts.sinked[threshold.parsed.AggregationMethod] = sinkImpl.P(threshold.parsed.AggregationValue.Float64 / 100)
+			key := fmt.Sprintf("p(%g)", threshold.parsed.AggregationValue.Float64)
+			ts.sinked[key] = sinkImpl.P(threshold.parsed.AggregationValue.Float64 / 100)
 		}
 	case *RateSink:
 		ts.sinked["rate"] = float64(sinkImpl.Trues) / float64(sinkImpl.Total)
@@ -244,6 +248,58 @@ func (ts *Thresholds) Parse() error {
 	return nil
 }
 
+// ErrInvalidThreshold indicates a threshold is not valid
+var ErrInvalidThreshold = errors.New("invalid threshold")
+
+// Validate ensures a threshold definition is consistent with the metric it applies to.
+// Given a metric registry and a metric name to apply the expressions too, Validate will
+// assert that each threshold expression uses an aggregation method that's supported by the
+// provided metric. It returns an error otherwise.
+// Note that this function expects the passed in thresholds to have been parsed already, and
+// have their Parsed (ThresholdExpression) field already filled.
+func (ts *Thresholds) Validate(metricName string, r *Registry) error {
+	parsedMetricName, _, err := ParseMetricName(metricName)
+	if err != nil {
+		err := fmt.Errorf("unable to validate threshold expressions; reason: %w", ErrMetricNameParsing)
+		return errext.WithExitCodeIfNone(err, exitcodes.InvalidConfig)
+	}
+
+	// Obtain the metric the thresholds apply to from the registry.
+	// if the metric doesn't exist, then we return an error indicating
+	// the InvalidConfig exitcode should be used.
+	metric := r.Get(parsedMetricName)
+	if metric == nil {
+		err := fmt.Errorf("%w defined on %s; reason: no metric name %q found", ErrInvalidThreshold, metricName, metricName)
+		return errext.WithExitCodeIfNone(err, exitcodes.InvalidConfig)
+	}
+
+	for _, threshold := range ts.Thresholds {
+		// Return a digestable error if we attempt to validate a threshold
+		// that hasn't been parsed yet.
+		if threshold.parsed == nil {
+			return fmt.Errorf("unable to validate threshold %q on metric %s; reason: "+
+				"threshold was not parsed", threshold.Source, metricName)
+		}
+
+		// If the threshold's expression aggregation method is not
+		// supported for the metric we validate against, then we return
+		// an error indicating the InvalidConfig exitcode should be used.
+		if !metric.Type.supportsAggregationMethod(threshold.parsed.AggregationMethod) {
+			err := fmt.Errorf(
+				"%w %q applied on metric %s; reason: "+
+					"unsupported aggregation method %s on metric of type %s. "+
+					"supported aggregation methods for this metric are: %s",
+				ErrInvalidThreshold, threshold.Source, metricName,
+				threshold.parsed.AggregationMethod, metric.Type,
+				strings.Join(metric.Type.supportedAggregationMethods(), ", "),
+			)
+			return errext.WithExitCodeIfNone(err, exitcodes.InvalidConfig)
+		}
+	}
+
+	return nil
+}
+
 // UnmarshalJSON is implementation of json.Unmarshaler
 func (ts *Thresholds) UnmarshalJSON(data []byte) error {
 	var configs []thresholdConfig

metrics/thresholds_parser.go

@@ -49,6 +49,28 @@ type thresholdExpression struct {
 	Value float64
 }
 
+// SinkKey computes the key used to index a thresholdExpression in the engine's sinks.
+//
+// During execution, the engine "sinks" metrics into a internal mapping, so that
+// thresholds can be asserted against them. This method is a helper to normalize the
+// sink the threshold expression should be applied to.
+//
+// Because a threshold expression's aggregation method can either be
+// a static keyword ("count", "rate", etc...), or a parametric
+// expression ("p(somefloatingpointvalue)"), we need to handle this
+// case specifically. If we encounter the percentile aggregation method token,
+// we recompute the whole "p(value)" expression in order to look for it in the
+// sinks.
+func (te *thresholdExpression) SinkKey() string {
+	//
+	sinkKey := te.AggregationMethod
+	if te.AggregationMethod == tokenPercentile {
+		sinkKey = fmt.Sprintf("%s(%g)", tokenPercentile, te.AggregationValue.Float64)
+	}
+
+	return sinkKey
+}
+
 // parseThresholdAssertion parses a threshold condition expression,
 // as defined in a JS script (for instance p(95)<1000), into a thresholdExpression
 // instance.
@@ -192,7 +214,7 @@ func parseThresholdAggregationMethod(input string) (string, null.Float, error) {
 		// Percentile expressions being of the form p(value),
 		// they won't be matched here.
 		if m == input {
-			return input, null.Float{}, nil
+			return m, null.Float{}, nil
 		}
 	}
 
@@ -203,7 +225,7 @@ func parseThresholdAggregationMethod(input string) (string, null.Float, error) {
 			return "", null.Float{}, fmt.Errorf("malformed percentile value; reason: %w", err)
 		}
 
-		return input, null.FloatFrom(aggregationValue), nil
+		return tokenPercentile, null.FloatFrom(aggregationValue), nil
 	}
 
 	return "", null.Float{}, fmt.Errorf("failed parsing method from expression")

metrics/thresholds_parser_test.go

@@ -103,63 +103,63 @@ func TestParseThresholdAggregationMethod(t *testing.T) {
 		{
 			name:            "count method is parsed",
 			input:           "count",
-			wantMethod:      "count",
+			wantMethod:      tokenCount,
 			wantMethodValue: null.Float{},
 			wantErr:         false,
 		},
 		{
 			name:            "rate method is parsed",
 			input:           "rate",
-			wantMethod:      "rate",
+			wantMethod:      tokenRate,
 			wantMethodValue: null.Float{},
 			wantErr:         false,
 		},
 		{
 			name:            "value method is parsed",
 			input:           "value",
-			wantMethod:      "value",
+			wantMethod:      tokenValue,
 			wantMethodValue: null.Float{},
 			wantErr:         false,
 		},
 		{
 			name:            "avg method is parsed",
 			input:           "avg",
-			wantMethod:      "avg",
+			wantMethod:      tokenAvg,
 			wantMethodValue: null.Float{},
 			wantErr:         false,
 		},
 		{
 			name:            "min method is parsed",
 			input:           "min",
-			wantMethod:      "min",
+			wantMethod:      tokenMin,
 			wantMethodValue: null.Float{},
 			wantErr:         false,
 		},
 		{
 			name:            "max method is parsed",
 			input:           "max",
-			wantMethod:      "max",
+			wantMethod:      tokenMax,
 			wantMethodValue: null.Float{},
 			wantErr:         false,
 		},
 		{
 			name:            "med method is parsed",
 			input:           "med",
-			wantMethod:      "med",
+			wantMethod:      tokenMed,
 			wantMethodValue: null.Float{},
 			wantErr:         false,
 		},
 		{
 			name:            "percentile method with integer value is parsed",
 			input:           "p(99)",
-			wantMethod:      "p(99)",
+			wantMethod:      tokenPercentile,
 			wantMethodValue: null.FloatFrom(99),
 			wantErr:         false,
 		},
 		{
 			name:            "percentile method with floating point value is parsed",
 			input:           "p(99.9)",
-			wantMethod:      "p(99.9)",
+			wantMethod:      tokenPercentile,
 			wantMethodValue: null.FloatFrom(99.9),
 			wantErr:         false,
 		},