Revamp the end-of-test summary

.github/workflows/xk6-tests/xk6-test.js

@@ -11,7 +11,7 @@ export let options = {
 
 export function handleSummary(data) {
     return {
-        'summary-results.txt': data.metrics.foos.values.count.toString(),
+        'summary-results.txt': data.metrics.custom.foos.values.count.toString(),
     };
 }
 

internal/cmd/outputs.go

@@ -37,6 +37,7 @@ const (
 	builtinOutputKafka
 	builtinOutputStatsd
 	builtinOutputExperimentalOpentelemetry
+	builtinOutputSummary
 )
 
 // TODO: move this to an output sub-module after we get rid of the old collectors?

internal/cmd/outputs_test.go

@@ -11,6 +11,7 @@ func TestBuiltinOutputString(t *testing.T) {
 	exp := []string{
 		"cloud", "csv", "datadog", "experimental-prometheus-rw",
 		"influxdb", "json", "kafka", "statsd", "experimental-opentelemetry",
+		"summary",
 	}
 	assert.Equal(t, exp, builtinOutputStrings())
 }

internal/cmd/run.go

@@ -25,6 +25,7 @@ import (
 	"go.k6.io/k6/internal/execution/local"
 	"go.k6.io/k6/internal/lib/trace"
 	"go.k6.io/k6/internal/metrics/engine"
+	"go.k6.io/k6/internal/output/summary"
 	"go.k6.io/k6/internal/ui/pb"
 	"go.k6.io/k6/js/common"
 	"go.k6.io/k6/lib"
@@ -189,26 +190,79 @@ func (c *cmdRun) run(cmd *cobra.Command, args []string) (err error) {
 	}
 
 	executionState := execScheduler.GetState()
-	if !testRunState.RuntimeOptions.NoSummary.Bool {
-		defer func() {
-			logger.Debug("Generating the end-of-test summary...")
-			summaryResult, hsErr := test.initRunner.HandleSummary(globalCtx, &lib.Summary{
-				Metrics:         metricsEngine.ObservedMetrics,
-				RootGroup:       testRunState.GroupSummary.Group(),
-				TestRunDuration: executionState.GetCurrentTestRunDuration(),
-				NoColor:         c.gs.Flags.NoColor,
-				UIState: lib.UIState{
-					IsStdOutTTY: c.gs.Stdout.IsTTY,
-					IsStdErrTTY: c.gs.Stderr.IsTTY,
-				},
+	if !testRunState.RuntimeOptions.NoSummary.Bool { //nolint:nestif
+		sm, err := lib.ValidateSummaryMode(testRunState.RuntimeOptions.SummaryMode.String)
+		if err != nil {
+			logger.WithError(err).Warnf(
+				"invalid summary mode %q, falling back to \"compact\" (default)",
+				testRunState.RuntimeOptions.SummaryMode.String,
+			)
+		}
+
+		switch sm {
+		// TODO: Remove this code block once we stop supporting the legacy summary, and just leave the default.
+		case lib.SummaryModeLegacy:
+			// At the end of the test run
+			defer func() {
+				logger.Debug("Generating the end-of-test summary...")
+
+				legacySummary := &lib.LegacySummary{
+					Metrics:         metricsEngine.ObservedMetrics,
+					RootGroup:       testRunState.GroupSummary.Group(),
+					TestRunDuration: executionState.GetCurrentTestRunDuration(),
+					NoColor:         c.gs.Flags.NoColor,
+					UIState: lib.UIState{
+						IsStdOutTTY: c.gs.Stdout.IsTTY,
+						IsStdErrTTY: c.gs.Stderr.IsTTY,
+					},
+				}
+
+				summaryResult, hsErr := test.initRunner.HandleSummary(globalCtx, legacySummary, nil)
+				if hsErr == nil {
+					hsErr = handleSummaryResult(c.gs.FS, c.gs.Stdout, c.gs.Stderr, summaryResult)
+				}
+				if hsErr != nil {
+					logger.WithError(hsErr).Error("failed to handle the end-of-test summary")
+				}
+			}()
+		default:
+			// Instantiates the summary output
+			summaryOutput, err := summary.New(output.Params{
+				RuntimeOptions: testRunState.RuntimeOptions,
+				Logger:         c.gs.Logger,
 			})
-			if hsErr == nil {
-				hsErr = handleSummaryResult(c.gs.FS, c.gs.Stdout, c.gs.Stderr, summaryResult)
-			}
-			if hsErr != nil {
-				logger.WithError(hsErr).Error("failed to handle the end-of-test summary")
+			if err != nil {
+				logger.WithError(err).Error("failed to initialize the end-of-test summary output")
 			}
-		}()
+			outputs = append(outputs, summaryOutput)
+
+			// At the end of the test run
+			defer func() {
+				logger.Debug("Generating the end-of-test summary...")
+
+				summary := summaryOutput.Summary(
+					executionState.GetCurrentTestRunDuration(),
+					metricsEngine.ObservedMetrics,
+					test.initRunner.GetOptions(),
+				)
+
+				// TODO: We should probably try to move these out of the summary,
+				// likely as an additional argument like options.
+				summary.NoColor = c.gs.Flags.NoColor
+				summary.UIState = lib.UIState{
+					IsStdOutTTY: c.gs.Stdout.IsTTY,
+					IsStdErrTTY: c.gs.Stderr.IsTTY,
+				}
+
+				summaryResult, hsErr := test.initRunner.HandleSummary(globalCtx, nil, summary)
+				if hsErr == nil {
+					hsErr = handleSummaryResult(c.gs.FS, c.gs.Stdout, c.gs.Stderr, summaryResult)
+				}
+				if hsErr != nil {
+					logger.WithError(hsErr).Error("failed to handle the end-of-test summary")
+				}
+			}()
+		}
 	}
 
 	waitInitDone := emitEvent(&event.Event{Type: event.Init})

internal/cmd/run_test.go

@@ -326,7 +326,7 @@ func TestThresholdsRuntimeBehavior(t *testing.T) {
 			name:              "#2518: submetrics without values should be rendered under their parent metric #2518",
 			testFilename:      "thresholds/thresholds_on_submetric_without_samples.js",
 			expExitCode:       0,
-			expStdoutContains: "     one..................: 0   0/s\n       { tag:xyz }........: 0   0/s\n",
+			expStdoutContains: "    one....................................: 0   0/s\n      { tag:xyz }..........................: 0   0/s\n",
 		},
 		{
 			name:         "#2512: parsing threshold names containing parsable tokens should be valid",
@@ -337,7 +337,7 @@ func TestThresholdsRuntimeBehavior(t *testing.T) {
 			name:                 "#2520: thresholds over metrics without values should avoid division by zero and displaying NaN values",
 			testFilename:         "thresholds/empty_sink_no_nan.js",
 			expExitCode:          0,
-			expStdoutContains:    "rate.................: 0.00%",
+			expStdoutContains:    "rate...................................: 0.00%",
 			expStdoutNotContains: "NaN",
 		},
 	}

internal/cmd/runtime_options.go

@@ -31,6 +31,8 @@ extended: base + sets "global" as alias for "globalThis"
 	flags.StringArrayP("env", "e", nil, "add/override environment variable with `VAR=value`")
 	flags.Bool("no-thresholds", false, "don't run thresholds")
 	flags.Bool("no-summary", false, "don't show the summary at the end of the test")
+	flags.String("with-summary", lib.SummaryModeCompact.String(), "determine the summary mode,"+
+		" \"compact\", \"full\" or \"legacy\"")
 	flags.String(
 		"summary-export",
 		"",
@@ -41,94 +43,119 @@ extended: base + sets "global" as alias for "globalThis"
 	return flags
 }
 
-func saveBoolFromEnv(env map[string]string, varName string, placeholder *null.Bool) error {
-	strValue, ok := env[varName]
-	if !ok {
-		return nil
+func getRuntimeOptions(
+	flags *pflag.FlagSet,
+	environment map[string]string,
+) (lib.RuntimeOptions, error) {
+	// TODO: refactor with composable helpers as a part of #883, to reduce copy-paste
+	// TODO: get these options out of the JSON config file as well?
+	opts, err := populateRuntimeOptionsFromEnv(runtimeOptionsFromFlags(flags), environment)
+	if err != nil {
+		return opts, err
 	}
-	val, err := strconv.ParseBool(strValue)
+
+	// Set/overwrite environment variables with custom user-supplied values
+	envVars, err := flags.GetStringArray("env")
 	if err != nil {
-		return fmt.Errorf("env var '%s' is not a valid boolean value: %w", varName, err)
+		return opts, err
 	}
-	// Only override if not explicitly set via the CLI flag
-	if !placeholder.Valid {
-		*placeholder = null.BoolFrom(val)
+
+	for _, kv := range envVars {
+		k, v := state.ParseEnvKeyValue(kv)
+		// Allow only alphanumeric ASCII variable names for now
+		if !userEnvVarName.MatchString(k) {
+			return opts, fmt.Errorf("invalid environment variable name '%s'", k)
+		}
+		opts.Env[k] = v
 	}
-	return nil
+
+	return opts, nil
 }
 
-func getRuntimeOptions(flags *pflag.FlagSet, environment map[string]string) (lib.RuntimeOptions, error) {
-	// TODO: refactor with composable helpers as a part of #883, to reduce copy-paste
-	// TODO: get these options out of the JSON config file as well?
+func runtimeOptionsFromFlags(flags *pflag.FlagSet) lib.RuntimeOptions {
 	opts := lib.RuntimeOptions{
 		TestType:             getNullString(flags, "type"),
 		IncludeSystemEnvVars: getNullBool(flags, "include-system-env-vars"),
 		CompatibilityMode:    getNullString(flags, "compatibility-mode"),
 		NoThresholds:         getNullBool(flags, "no-thresholds"),
 		NoSummary:            getNullBool(flags, "no-summary"),
+		SummaryMode:          getNullString(flags, "with-summary"),
 		SummaryExport:        getNullString(flags, "summary-export"),
 		TracesOutput:         getNullString(flags, "traces-output"),
 		Env:                  make(map[string]string),
 	}
+	return opts
+}
+
+func populateRuntimeOptionsFromEnv(opts lib.RuntimeOptions, environment map[string]string) (lib.RuntimeOptions, error) {
+	// Only override if not explicitly set via the CLI flag
 
-	if envVar, ok := environment["K6_TYPE"]; ok && !opts.TestType.Valid {
-		// Only override if not explicitly set via the CLI flag
+	if envVar, ok := environment["K6_TYPE"]; !opts.TestType.Valid && ok {
 		opts.TestType = null.StringFrom(envVar)
 	}
-	if envVar, ok := environment["K6_COMPATIBILITY_MODE"]; ok && !opts.CompatibilityMode.Valid {
-		// Only override if not explicitly set via the CLI flag
+
+	if envVar, ok := environment["K6_COMPATIBILITY_MODE"]; !opts.CompatibilityMode.Valid && ok {
 		opts.CompatibilityMode = null.StringFrom(envVar)
 	}
-	if _, err := lib.ValidateCompatibilityMode(opts.CompatibilityMode.String); err != nil {
-		// some early validation
-		return opts, err
+
+	if envVar, ok := environment["K6_WITH_SUMMARY"]; !opts.SummaryMode.Valid && ok {
+		opts.SummaryMode = null.StringFrom(envVar)
 	}
 
 	if err := saveBoolFromEnv(environment, "K6_INCLUDE_SYSTEM_ENV_VARS", &opts.IncludeSystemEnvVars); err != nil {
 		return opts, err
 	}
+
 	if err := saveBoolFromEnv(environment, "K6_NO_THRESHOLDS", &opts.NoThresholds); err != nil {
 		return opts, err
 	}
+
 	if err := saveBoolFromEnv(environment, "K6_NO_SUMMARY", &opts.NoSummary); err != nil {
 		return opts, err
 	}
 
-	if envVar, ok := environment["K6_SUMMARY_EXPORT"]; ok {
-		if !opts.SummaryExport.Valid {
-			opts.SummaryExport = null.StringFrom(envVar)
-		}
+	if _, err := lib.ValidateCompatibilityMode(opts.CompatibilityMode.String); err != nil {
+		// some early validation
+		return opts, err
 	}
 
-	if envVar, ok := environment["SSLKEYLOGFILE"]; ok {
-		if !opts.KeyWriter.Valid {
-			opts.KeyWriter = null.StringFrom(envVar)
-		}
+	if _, err := lib.ValidateSummaryMode(opts.SummaryMode.String); err != nil {
+		// some early validation
+		return opts, err
 	}
 
-	if envVar, ok := environment["K6_TRACES_OUTPUT"]; ok {
-		if !opts.TracesOutput.Valid {
-			opts.TracesOutput = null.StringFrom(envVar)
-		}
+	if envVar, ok := environment["K6_SUMMARY_EXPORT"]; !opts.SummaryExport.Valid && ok {
+		opts.SummaryExport = null.StringFrom(envVar)
 	}
 
-	if opts.IncludeSystemEnvVars.Bool { // If enabled, gather the actual system environment variables
-		opts.Env = environment
+	if envVar, ok := environment["SSLKEYLOGFILE"]; !opts.KeyWriter.Valid && ok {
+		opts.KeyWriter = null.StringFrom(envVar)
 	}
 
-	// Set/overwrite environment variables with custom user-supplied values
-	envVars, err := flags.GetStringArray("env")
-	if err != nil {
-		return opts, err
+	if envVar, ok := environment["K6_TRACES_OUTPUT"]; !opts.TracesOutput.Valid && ok {
+		opts.TracesOutput = null.StringFrom(envVar)
 	}
-	for _, kv := range envVars {
-		k, v := state.ParseEnvKeyValue(kv)
-		// Allow only alphanumeric ASCII variable names for now
-		if !userEnvVarName.MatchString(k) {
-			return opts, fmt.Errorf("invalid environment variable name '%s'", k)
-		}
-		opts.Env[k] = v
+
+	// If enabled, gather the actual system environment variables
+	if opts.IncludeSystemEnvVars.Bool {
+		opts.Env = environment
 	}
 
 	return opts, nil
 }
+
+func saveBoolFromEnv(env map[string]string, varName string, placeholder *null.Bool) error {
+	strValue, ok := env[varName]
+	if !ok {
+		return nil
+	}
+	val, err := strconv.ParseBool(strValue)
+	if err != nil {
+		return fmt.Errorf("env var '%s' is not a valid boolean value: %w", varName, err)
+	}
+	// Only override if not explicitly set via the CLI flag
+	if !placeholder.Valid {
+		*placeholder = null.BoolFrom(val)
+	}
+	return nil
+}