From 665167fec1df81ef55fa621947ace8b506851f8d Mon Sep 17 00:00:00 2001 From: Nedyalko Andreev Date: Sun, 4 Dec 2022 17:20:39 +0200 Subject: [PATCH 1/9] Fix bug in integration test harness code The cleanup function would have never been called, since expectedExitCode was always 0 in the constructor. --- cmd/root_test.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cmd/root_test.go b/cmd/root_test.go index baaf7984a3f..d53faddc923 100644 --- a/cmd/root_test.go +++ b/cmd/root_test.go @@ -79,16 +79,16 @@ func newGlobalTestState(t *testing.T) *globalTestState { defaultOsExitHandle := func(exitCode int) { cancel() osExitCalled = true - require.Equal(t, ts.expectedExitCode, exitCode) + assert.Equal(t, ts.expectedExitCode, exitCode) } - if ts.expectedExitCode > 0 { - // Ensure that, if we expected to receive an error, our `os.Exit()` mock - // function was actually called. - t.Cleanup(func() { - assert.True(t, osExitCalled) - }) - } + t.Cleanup(func() { + if ts.expectedExitCode > 0 { + // Ensure that, if we expected to receive an error, our `os.Exit()` mock + // function was actually called. + assert.Truef(t, osExitCalled, "expected exit code %d, but the os.Exit() mock was not called", ts.expectedExitCode) + } + }) outMutex := &sync.Mutex{} defaultFlags := getDefaultFlags(".config") From 17e6bba185c6143de8ab38de1cfa94a189230137 Mon Sep 17 00:00:00 2001 From: Nedyalko Andreev Date: Sun, 4 Dec 2022 17:22:30 +0200 Subject: [PATCH 2/9] Add an integration test for setup data and metric tag isolation --- cmd/integration_test.go | 100 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 97 insertions(+), 3 deletions(-) diff --git a/cmd/integration_test.go b/cmd/integration_test.go index 7767cd7eb23..72d5a68e94d 100644 --- a/cmd/integration_test.go +++ b/cmd/integration_test.go @@ -804,6 +804,100 @@ func TestAbortedByScriptInitError(t *testing.T) { require.Contains(t, stdOut, `level=debug msg="Sending test finished" output=cloud ref=111 run_status=7 tainted=false`) } -// TODO: add an integration test that verifies that unindexable tags work as -// expected and that VU tags from different scenarios don't cross between -// scenarios and pollute other metrics. +func TestMetricTagAndSetupDataIsolation(t *testing.T) { + t.Parallel() + script := []byte(` + import exec from 'k6/execution'; + import { Counter } from 'k6/metrics'; + import { sleep } from 'k6'; + + export const options = { + scenarios: { + sc1: { + executor: 'shared-iterations', + vus: 2, + iterations: 20, + maxDuration: '7s', + gracefulStop: 0, + exec: 'sc1', + }, + sc2: { + executor: 'per-vu-iterations', + vus: 1, + iterations: 1, + startTime: '7s', + exec: 'sc2', + }, + }, + thresholds: { + 'iterations': ['count == 21'], + 'iterations{scenario:sc1}': ['count == 20'], + 'iterations{sc:1}': ['count == 20'], + 'iterations{scenario:sc2}': ['count == 1'], + 'mycounter': ['count == 23'], + 'mycounter{sc:1}': ['count == 20'], + 'mycounter{setup:true}': ['count == 1'], + 'mycounter{myiter:1}': ['count >= 1', 'count <= 2'], + 'mycounter{myiter:2}': ['count >= 1', 'count <= 2'], + 'mycounter{scenario:sc2}': ['count == 1'], + 'mycounter{scenario:sc2,sc:1}': ['count == 0'], + 'vus_max': ['value == 2'], + }, + }; + let myCounter = new Counter('mycounter'); + + export function setup() { + exec.vu.tags.setup = 'true'; + myCounter.add(1); + return { v: 0 }; + } + + export function sc1(data) { + if (data.v !== __ITER) { + throw new Error('sc1: wrong data for iter ' + __ITER + ': ' + JSON.stringify(data)); + } + if (__ITER != 0 && data.v != exec.vu.tags.myiter) { + throw new Error('sc1: wrong vu tags for iter ' + __ITER + ': ' + JSON.stringify(exec.vu.tags)); + } + data.v += 1; + exec.vu.tags.myiter = data.v; + exec.vu.tags.sc = 1; + myCounter.add(1); + sleep(0.02); // encourage using both of the VUs + } + + export function sc2(data) { + if (data.v === 0) { + throw new Error('sc2: wrong data, expected VU to have modified setup data locally: ' + data.v); + } + + if (typeof exec.vu.tags.myiter !== 'undefined') { + throw new Error( + 'sc2: wrong tags, expected VU to have new tags in new scenario: ' + JSON.stringify(exec.vu.tags), + ); + } + + myCounter.add(1); + } + + export function teardown(data) { + if (data.v !== 0) { + throw new Error('teardown: wrong data: ' + data.v); + } + myCounter.add(1); + } + `) + + srv := getCloudTestEndChecker(t, lib.RunStatusFinished, cloudapi.ResultStatusPassed) + + ts := newGlobalTestState(t) + require.NoError(t, afero.WriteFile(ts.fs, filepath.Join(ts.cwd, "test.js"), script, 0o644)) + ts.envVars = map[string]string{"K6_CLOUD_HOST": srv.URL} + ts.args = []string{"k6", "run", "--quiet", "--log-output=stdout", "--out", "cloud", "test.js"} + + newRootCommand(ts.globalState).execute() + + stdOut := ts.stdOut.String() + t.Log(stdOut) + require.Equal(t, 12, strings.Count(stdOut, "✓")) +} From 3a88fdce338977190f6f82eec3063f79a45a21f9 Mon Sep 17 00:00:00 2001 From: Nedyalko Andreev Date: Sun, 4 Dec 2022 20:30:53 +0200 Subject: [PATCH 3/9] Add an integration test duplicate for the Engine's TestActiveVUsCount --- cmd/integration_test.go | 142 ++++++++++++++++++++++++++++++++++++++++ core/engine_test.go | 1 + 2 files changed, 143 insertions(+) diff --git a/cmd/integration_test.go b/cmd/integration_test.go index 72d5a68e94d..5a294f18d71 100644 --- a/cmd/integration_test.go +++ b/cmd/integration_test.go @@ -901,3 +901,145 @@ func TestMetricTagAndSetupDataIsolation(t *testing.T) { t.Log(stdOut) require.Equal(t, 12, strings.Count(stdOut, "✓")) } + +func getSampleValues(t *testing.T, jsonOutput []byte, metric string, tags map[string]string) []float64 { + jsonLines := bytes.Split(jsonOutput, []byte("\n")) + result := []float64{} + + tagsMatch := func(rawTags interface{}) bool { + sampleTags, ok := rawTags.(map[string]interface{}) + require.True(t, ok) + for k, v := range tags { + rv, sok := sampleTags[k] + if !sok { + return false + } + rvs, sok := rv.(string) + require.True(t, sok) + if v != rvs { + return false + } + } + return true + } + + for _, jsonLine := range jsonLines { + if len(jsonLine) == 0 { + continue + } + var line map[string]interface{} + require.NoError(t, json.Unmarshal(jsonLine, &line)) + sampleType, ok := line["type"].(string) + require.True(t, ok) + if sampleType != "Point" { + continue + } + sampleMetric, ok := line["metric"].(string) + require.True(t, ok) + if sampleMetric != metric { + continue + } + sampleData, ok := line["data"].(map[string]interface{}) + require.True(t, ok) + + if !tagsMatch(sampleData["tags"]) { + continue + } + + samplValue, ok := sampleData["value"].(float64) + require.True(t, ok) + result = append(result, samplValue) + } + + return result +} + +func sum(vals []float64) (sum float64) { + for _, val := range vals { + sum += val + } + return sum +} + +func max(vals []float64) float64 { + max := vals[0] + for _, val := range vals { + if max < val { + max = val + } + } + return max +} + +func TestActiveVUsCount(t *testing.T) { + t.Parallel() + + script := []byte(` + var sleep = require('k6').sleep; + + exports.options = { + scenarios: { + carr1: { + executor: 'constant-arrival-rate', + rate: 10, + preAllocatedVUs: 1, + maxVUs: 10, + startTime: '0s', + duration: '3s', + gracefulStop: '0s', + }, + carr2: { + executor: 'constant-arrival-rate', + rate: 10, + preAllocatedVUs: 1, + maxVUs: 10, + duration: '3s', + startTime: '3s', + gracefulStop: '0s', + }, + rarr: { + executor: 'ramping-arrival-rate', + startRate: 5, + stages: [ + { target: 10, duration: '2s' }, + { target: 0, duration: '2s' }, + ], + preAllocatedVUs: 1, + maxVUs: 10, + startTime: '6s', + gracefulStop: '0s', + }, + } + } + + exports.default = function () { + sleep(5); + } + `) + + ts := newGlobalTestState(t) + require.NoError(t, afero.WriteFile(ts.fs, filepath.Join(ts.cwd, "test.js"), script, 0o644)) + ts.args = []string{"k6", "run", "--compatibility-mode", "base", "--out", "json=results.json", "test.js"} + newRootCommand(ts.globalState).execute() + + stdOut := ts.stdOut.String() + t.Log(stdOut) + + jsonResults, err := afero.ReadFile(ts.fs, "results.json") + require.NoError(t, err) + // t.Log(string(jsonResults)) + assert.Equal(t, float64(10), max(getSampleValues(t, jsonResults, "vus_max", nil))) + assert.Equal(t, float64(10), max(getSampleValues(t, jsonResults, "vus", nil))) + assert.Equal(t, float64(0), sum(getSampleValues(t, jsonResults, "iterations", nil))) + + logEntries := ts.loggerHook.Drain() + assert.Len(t, logEntries, 4) + for i, logEntry := range logEntries { + assert.Equal(t, logrus.WarnLevel, logEntry.Level) + if i < 3 { + assert.Equal(t, "Insufficient VUs, reached 10 active VUs and cannot initialize more", logEntry.Message) + } else { + assert.Equal(t, "No script iterations finished, consider making the test duration longer", logEntry.Message) + } + } +} diff --git a/core/engine_test.go b/core/engine_test.go index d963d29e0f4..e7faaee7bc3 100644 --- a/core/engine_test.go +++ b/core/engine_test.go @@ -1231,6 +1231,7 @@ func TestEngineRunsTeardownEvenAfterTestRunIsAborted(t *testing.T) { assert.Equal(t, 1.0, count) } +// TODO: delete, functionality duplicated in cmd/integration_test.go func TestActiveVUsCount(t *testing.T) { t.Parallel() From 7fedfd2c0adac206696933e2330d7189b9deda25 Mon Sep 17 00:00:00 2001 From: Nedyalko Andreev Date: Sun, 4 Dec 2022 20:43:59 +0200 Subject: [PATCH 4/9] Improve tests to check teardown() is called even when test run is aborted This is a simplified and better alternative to the existing TestEngineRunsTeardownEvenAfterTestRunIsAborted test for the Engine. --- cmd/integration_test.go | 26 ++++++++++++++++++++++++-- core/engine_test.go | 1 + 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/cmd/integration_test.go b/cmd/integration_test.go index 5a294f18d71..93e591b22a1 100644 --- a/cmd/integration_test.go +++ b/cmd/integration_test.go @@ -578,6 +578,10 @@ func TestAbortedByThreshold(t *testing.T) { }; export default function () {}; + + export function teardown() { + console.log('teardown() called'); + } `) srv := getCloudTestEndChecker(t, lib.RunStatusAbortedThreshold, cloudapi.ResultStatusFailed) @@ -593,13 +597,16 @@ func TestAbortedByThreshold(t *testing.T) { assert.True(t, testutils.LogContains(ts.loggerHook.Drain(), logrus.ErrorLevel, `some thresholds have failed`)) stdOut := ts.stdOut.String() t.Log(stdOut) - require.Contains(t, stdOut, `✗ iterations...........: `) - require.Contains(t, stdOut, `level=debug msg="Sending test finished" output=cloud ref=111 run_status=8 tainted=true`) + assert.Contains(t, stdOut, `✗ iterations`) + assert.Contains(t, stdOut, `teardown() called`) + assert.Contains(t, stdOut, `level=debug msg="Sending test finished" output=cloud ref=111 run_status=8 tainted=true`) } func TestAbortedByUserWithGoodThresholds(t *testing.T) { t.Parallel() script := []byte(` + import { Counter } from 'k6/metrics'; + export const options = { scenarios: { sc1: { @@ -611,9 +618,17 @@ func TestAbortedByUserWithGoodThresholds(t *testing.T) { }, thresholds: { 'iterations': ['count >= 1'], + 'tc': ['count == 1'], + 'tc{group:::setup}': ['count == 0'], + 'tc{group:::teardown}': ['count == 1'], }, }; + let tc = new Counter('tc'); + export function teardown() { + tc.add(1); + } + export default function () {}; `) @@ -639,6 +654,8 @@ func TestAbortedByUserWithGoodThresholds(t *testing.T) { stdOut := ts.stdOut.String() t.Log(stdOut) require.Contains(t, stdOut, `✓ iterations`) + require.Contains(t, stdOut, `✓ tc`) + require.Contains(t, stdOut, `✓ { group:::teardown }`) require.Contains(t, stdOut, `Stopping k6 in response to signal`) require.Contains(t, stdOut, `level=debug msg="Sending test finished" output=cloud ref=111 run_status=5 tainted=false`) } @@ -651,6 +668,10 @@ func TestAbortedByUserWithRestAPI(t *testing.T) { console.log('a simple iteration') sleep(1); }; + + export function teardown() { + console.log('teardown() called'); + } `) srv := getCloudTestEndChecker(t, lib.RunStatusAbortedUser, cloudapi.ResultStatusPassed) @@ -701,6 +722,7 @@ func TestAbortedByUserWithRestAPI(t *testing.T) { stdOut := ts.stdOut.String() t.Log(stdOut) require.Contains(t, stdOut, `a simple iteration`) + require.Contains(t, stdOut, `teardown() called`) require.Contains(t, stdOut, `PATCH /v1/status`) require.Contains(t, stdOut, `run: stopped by user; exiting...`) require.Contains(t, stdOut, `level=debug msg="Sending test finished" output=cloud ref=111 run_status=5 tainted=false`) diff --git a/core/engine_test.go b/core/engine_test.go index e7faaee7bc3..500a79df545 100644 --- a/core/engine_test.go +++ b/core/engine_test.go @@ -1188,6 +1188,7 @@ func TestMinIterationDurationInSetupTeardownStage(t *testing.T) { } } +// TODO: delete, functionality duplicated in cmd/integration_test.go func TestEngineRunsTeardownEvenAfterTestRunIsAborted(t *testing.T) { t.Parallel() From e1491c46420f86d3b0a49ab8697702fe2f3f25d3 Mon Sep 17 00:00:00 2001 From: Nedyalko Andreev Date: Sun, 4 Dec 2022 21:18:02 +0200 Subject: [PATCH 5/9] Use assert instead of require for non-critical checks in some tests --- cmd/integration_test.go | 60 ++++++++++++++++++++--------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/cmd/integration_test.go b/cmd/integration_test.go index 93e591b22a1..102e3d823b8 100644 --- a/cmd/integration_test.go +++ b/cmd/integration_test.go @@ -302,7 +302,7 @@ func testSSLKEYLOGFILE(t *testing.T, ts *globalTestState, filePath string) { sslloglines, err := afero.ReadFile(ts.fs, filepath.Join(ts.cwd, "ssl.log")) require.NoError(t, err) // TODO maybe have multiple depending on the ciphers used as that seems to change it - require.Regexp(t, "^CLIENT_[A-Z_]+ [0-9a-f]+ [0-9a-f]+\n", string(sslloglines)) + assert.Regexp(t, "^CLIENT_[A-Z_]+ [0-9a-f]+ [0-9a-f]+\n", string(sslloglines)) } func TestThresholdDeprecationWarnings(t *testing.T) { @@ -392,8 +392,8 @@ func TestSubMetricThresholdNoData(t *testing.T) { newRootCommand(ts.globalState).execute() - require.Len(t, ts.loggerHook.Drain(), 0) - require.Contains(t, ts.stdOut.String(), ` + assert.Len(t, ts.loggerHook.Drain(), 0) + assert.Contains(t, ts.stdOut.String(), ` one..................: 0 0/s { tag:xyz }........: 0 0/s two..................: 42`) @@ -424,7 +424,7 @@ func getCloudTestEndChecker(t *testing.T, expRunStatus lib.RunStatus, expResultS "POST ^/v1/tests$": http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { resp.WriteHeader(http.StatusOK) _, err := fmt.Fprintf(resp, `{"reference_id": "111"}`) - require.NoError(t, err) + assert.NoError(t, err) }), "POST ^/v1/tests/111$": http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { require.NotNil(t, req.Body) @@ -504,11 +504,11 @@ func TestSetupTeardownThresholds(t *testing.T) { newRootCommand(ts.globalState).execute() - require.Len(t, ts.loggerHook.Drain(), 0) + assert.Len(t, ts.loggerHook.Drain(), 0) stdOut := ts.stdOut.String() - require.Contains(t, stdOut, `✓ http_reqs......................: 7`) - require.Contains(t, stdOut, `✓ iterations.....................: 5`) - require.Contains(t, stdOut, `✓ setup_teardown.................: 2`) + assert.Contains(t, stdOut, `✓ http_reqs......................: 7`) + assert.Contains(t, stdOut, `✓ iterations.....................: 5`) + assert.Contains(t, stdOut, `✓ setup_teardown.................: 2`) } func TestThresholdsFailed(t *testing.T) { @@ -551,10 +551,10 @@ func TestThresholdsFailed(t *testing.T) { assert.True(t, testutils.LogContains(ts.loggerHook.Drain(), logrus.ErrorLevel, `some thresholds have failed`)) stdOut := ts.stdOut.String() t.Log(stdOut) - require.Contains(t, stdOut, ` ✓ iterations...........: 3`) - require.Contains(t, stdOut, ` ✗ { scenario:sc1 }...: 1`) - require.Contains(t, stdOut, ` ✗ { scenario:sc2 }...: 2`) - require.Contains(t, stdOut, ` ✓ { scenario:sc3 }...: 0 0/s`) + assert.Contains(t, stdOut, ` ✓ iterations...........: 3`) + assert.Contains(t, stdOut, ` ✗ { scenario:sc1 }...: 1`) + assert.Contains(t, stdOut, ` ✗ { scenario:sc2 }...: 2`) + assert.Contains(t, stdOut, ` ✓ { scenario:sc3 }...: 0 0/s`) } func TestAbortedByThreshold(t *testing.T) { @@ -653,11 +653,11 @@ func TestAbortedByUserWithGoodThresholds(t *testing.T) { assert.False(t, testutils.LogContains(ts.loggerHook.Drain(), logrus.ErrorLevel, `some thresholds have failed`)) stdOut := ts.stdOut.String() t.Log(stdOut) - require.Contains(t, stdOut, `✓ iterations`) - require.Contains(t, stdOut, `✓ tc`) - require.Contains(t, stdOut, `✓ { group:::teardown }`) - require.Contains(t, stdOut, `Stopping k6 in response to signal`) - require.Contains(t, stdOut, `level=debug msg="Sending test finished" output=cloud ref=111 run_status=5 tainted=false`) + assert.Contains(t, stdOut, `✓ iterations`) + assert.Contains(t, stdOut, `✓ tc`) + assert.Contains(t, stdOut, `✓ { group:::teardown }`) + assert.Contains(t, stdOut, `Stopping k6 in response to signal`) + assert.Contains(t, stdOut, `level=debug msg="Sending test finished" output=cloud ref=111 run_status=5 tainted=false`) } func TestAbortedByUserWithRestAPI(t *testing.T) { @@ -721,11 +721,11 @@ func TestAbortedByUserWithRestAPI(t *testing.T) { wg.Wait() stdOut := ts.stdOut.String() t.Log(stdOut) - require.Contains(t, stdOut, `a simple iteration`) - require.Contains(t, stdOut, `teardown() called`) - require.Contains(t, stdOut, `PATCH /v1/status`) - require.Contains(t, stdOut, `run: stopped by user; exiting...`) - require.Contains(t, stdOut, `level=debug msg="Sending test finished" output=cloud ref=111 run_status=5 tainted=false`) + assert.Contains(t, stdOut, `a simple iteration`) + assert.Contains(t, stdOut, `teardown() called`) + assert.Contains(t, stdOut, `PATCH /v1/status`) + assert.Contains(t, stdOut, `run: stopped by user; exiting...`) + assert.Contains(t, stdOut, `level=debug msg="Sending test finished" output=cloud ref=111 run_status=5 tainted=false`) } func TestAbortedByScriptSetupError(t *testing.T) { @@ -758,9 +758,9 @@ func TestAbortedByScriptSetupError(t *testing.T) { ts.outMutex.Unlock() t.Log(stdOut) - require.Contains(t, stdOut, `wonky setup`) - require.Contains(t, stdOut, `Error: foo`) - require.Contains(t, stdOut, `level=debug msg="Sending test finished" output=cloud ref=111 run_status=7 tainted=false`) + assert.Contains(t, stdOut, `wonky setup`) + assert.Contains(t, stdOut, `Error: foo`) + assert.Contains(t, stdOut, `level=debug msg="Sending test finished" output=cloud ref=111 run_status=7 tainted=false`) } func TestAbortedByScriptAbort(t *testing.T) { @@ -784,8 +784,8 @@ func TestAbortedByScriptAbort(t *testing.T) { stdOut := ts.stdOut.String() t.Log(stdOut) - require.Contains(t, stdOut, "test aborted: foo") - require.Contains(t, stdOut, `level=debug msg="Sending test finished" output=cloud ref=111 run_status=5 tainted=false`) + assert.Contains(t, stdOut, "test aborted: foo") + assert.Contains(t, stdOut, `level=debug msg="Sending test finished" output=cloud ref=111 run_status=5 tainted=false`) } func TestAbortedByScriptInitError(t *testing.T) { @@ -822,8 +822,8 @@ func TestAbortedByScriptInitError(t *testing.T) { ts.outMutex.Unlock() t.Log(stdOut) - require.Contains(t, stdOut, `Error: foo`) - require.Contains(t, stdOut, `level=debug msg="Sending test finished" output=cloud ref=111 run_status=7 tainted=false`) + assert.Contains(t, stdOut, `Error: foo`) + assert.Contains(t, stdOut, `level=debug msg="Sending test finished" output=cloud ref=111 run_status=7 tainted=false`) } func TestMetricTagAndSetupDataIsolation(t *testing.T) { @@ -921,7 +921,7 @@ func TestMetricTagAndSetupDataIsolation(t *testing.T) { stdOut := ts.stdOut.String() t.Log(stdOut) - require.Equal(t, 12, strings.Count(stdOut, "✓")) + assert.Equal(t, 12, strings.Count(stdOut, "✓")) } func getSampleValues(t *testing.T, jsonOutput []byte, metric string, tags map[string]string) []float64 { From a67cb56c423f58a4a38e5363dcaf15b3fcd731d0 Mon Sep 17 00:00:00 2001 From: Nedyalko Andreev Date: Sun, 4 Dec 2022 21:49:44 +0200 Subject: [PATCH 6/9] Add an integration test to verify minIterationDuration behavior --- cmd/integration_test.go | 42 +++++++++++++++++++++++++++++++++++++++++ core/engine_test.go | 1 + 2 files changed, 43 insertions(+) diff --git a/cmd/integration_test.go b/cmd/integration_test.go index 102e3d823b8..bf7133cdb8e 100644 --- a/cmd/integration_test.go +++ b/cmd/integration_test.go @@ -1065,3 +1065,45 @@ func TestActiveVUsCount(t *testing.T) { } } } + +func TestMinIterationDuration(t *testing.T) { + t.Parallel() + script := []byte(` + import { Counter } from 'k6/metrics'; + + export let options = { + minIterationDuration: '5s', + setupTimeout: '2s', + teardownTimeout: '2s', + thresholds: { + 'test_counter': ['count == 3'], + }, + }; + + var c = new Counter('test_counter'); + + export function setup() { c.add(1); }; + export default function () { c.add(1); }; + export function teardown() { c.add(1); }; + `) + + srv := getCloudTestEndChecker(t, lib.RunStatusFinished, cloudapi.ResultStatusPassed) + + ts := newGlobalTestState(t) + require.NoError(t, afero.WriteFile(ts.fs, filepath.Join(ts.cwd, "test.js"), script, 0o644)) + ts.envVars = map[string]string{"K6_CLOUD_HOST": srv.URL} + ts.args = []string{"k6", "run", "--quiet", "--log-output=stdout", "--out", "cloud", "test.js"} + + start := time.Now() + newRootCommand(ts.globalState).execute() + elapsed := time.Since(start) + assert.Greater(t, elapsed, 5*time.Second, "expected more time to have passed because of minIterationDuration") + assert.Less( + t, elapsed, 10*time.Second, + "expected less time to have passed because minIterationDuration should not affect setup() and teardown() ", + ) + + stdOut := ts.stdOut.String() + t.Log(stdOut) + assert.Contains(t, stdOut, "✓ test_counter.........: 3") +} diff --git a/core/engine_test.go b/core/engine_test.go index 500a79df545..63da02df2c9 100644 --- a/core/engine_test.go +++ b/core/engine_test.go @@ -1108,6 +1108,7 @@ func TestMetricsEmission(t *testing.T) { } } +// TODO: delete, functionality duplicated in cmd/integration_test.go func TestMinIterationDurationInSetupTeardownStage(t *testing.T) { t.Parallel() setupScript := ` From 690fbbb91d81ca28d0d994d9248aba6de0356673 Mon Sep 17 00:00:00 2001 From: Nedyalko Andreev Date: Sun, 4 Dec 2022 22:14:51 +0200 Subject: [PATCH 7/9] Enhance some integration tests to make 2 Engine tests redundant --- cmd/integration_test.go | 37 +++++++++++++++++++++++++++---------- core/engine_test.go | 2 ++ 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/cmd/integration_test.go b/cmd/integration_test.go index bf7133cdb8e..0a621d58c1e 100644 --- a/cmd/integration_test.go +++ b/cmd/integration_test.go @@ -730,19 +730,29 @@ func TestAbortedByUserWithRestAPI(t *testing.T) { func TestAbortedByScriptSetupError(t *testing.T) { t.Parallel() - script := []byte(` + depScript := []byte(` + export default function () { + baz(); + } + function baz() { + throw new Error("baz"); + } + `) + mainScript := []byte(` + import bar from "./bar.js"; export function setup() { console.log('wonky setup'); - throw new Error('foo'); - } - - export default function () {}; + bar(); + }; + export default function() {}; `) srv := getCloudTestEndChecker(t, lib.RunStatusAbortedScriptError, cloudapi.ResultStatusPassed) ts := newGlobalTestState(t) - require.NoError(t, afero.WriteFile(ts.fs, filepath.Join(ts.cwd, "test.js"), script, 0o644)) + require.NoError(t, afero.WriteFile(ts.fs, filepath.Join(ts.cwd, "test.js"), mainScript, 0o644)) + require.NoError(t, afero.WriteFile(ts.fs, filepath.Join(ts.cwd, "bar.js"), depScript, 0o644)) + ts.envVars = map[string]string{"K6_CLOUD_HOST": srv.URL} ts.args = []string{"k6", "run", "-v", "--out", "cloud", "--log-output=stdout", "test.js"} ts.expectedExitCode = int(exitcodes.ScriptException) @@ -759,7 +769,13 @@ func TestAbortedByScriptSetupError(t *testing.T) { t.Log(stdOut) assert.Contains(t, stdOut, `wonky setup`) - assert.Contains(t, stdOut, `Error: foo`) + + rootPath := "file:///" + if runtime.GOOS == "windows" { + rootPath += "c:/" + } + assert.Contains(t, stdOut, `level=error msg="Error: baz\n\tat baz (`+rootPath+`test/bar.js:6:9(3))\n\tat `+ + rootPath+`test/bar.js:3:3(3)\n\tat setup (`+rootPath+`test/test.js:5:3(9))\n\tat native\n" hint="script exception"`) assert.Contains(t, stdOut, `level=debug msg="Sending test finished" output=cloud ref=111 run_status=7 tainted=false`) } @@ -796,8 +812,8 @@ func TestAbortedByScriptInitError(t *testing.T) { iterations: 10, }; - if (__VU > 3) { - throw new Error('foo'); + if (__VU == 2) { + throw new Error('oops in ' + __VU); } export default function () {}; @@ -822,7 +838,8 @@ func TestAbortedByScriptInitError(t *testing.T) { ts.outMutex.Unlock() t.Log(stdOut) - assert.Contains(t, stdOut, `Error: foo`) + assert.Contains(t, stdOut, `level=error msg="Error: oops in 2\n\tat file:///`) + assert.Contains(t, stdOut, `hint="error while initializing VU #2 (script exception)"`) assert.Contains(t, stdOut, `level=debug msg="Sending test finished" output=cloud ref=111 run_status=7 tainted=false`) } diff --git a/core/engine_test.go b/core/engine_test.go index 63da02df2c9..9fce6be2613 100644 --- a/core/engine_test.go +++ b/core/engine_test.go @@ -840,6 +840,7 @@ func TestRunTags(t *testing.T) { } } +// TODO: delete, functionality duplicated in cmd/integration_test.go func TestSetupException(t *testing.T) { t.Parallel() @@ -892,6 +893,7 @@ func TestSetupException(t *testing.T) { } } +// TODO: delete, functionality duplicated in cmd/integration_test.go func TestVuInitException(t *testing.T) { t.Parallel() From f18a3bef6ed0a91fadaec234dbc021a1e31d6bd5 Mon Sep 17 00:00:00 2001 From: Nedyalko Andreev Date: Sun, 4 Dec 2022 23:31:54 +0200 Subject: [PATCH 8/9] Add an integration test that verifies per-testrun tags --- cmd/integration_test.go | 109 ++++++++++++++++++++++++++++++++++++++++ core/engine_test.go | 1 + 2 files changed, 110 insertions(+) diff --git a/cmd/integration_test.go b/cmd/integration_test.go index 0a621d58c1e..964c288d784 100644 --- a/cmd/integration_test.go +++ b/cmd/integration_test.go @@ -1124,3 +1124,112 @@ func TestMinIterationDuration(t *testing.T) { t.Log(stdOut) assert.Contains(t, stdOut, "✓ test_counter.........: 3") } + +func TestRunTags(t *testing.T) { + t.Parallel() + + tb := httpmultibin.NewHTTPMultiBin(t) + script := []byte(tb.Replacer.Replace(` + import http from 'k6/http'; + import ws from 'k6/ws'; + import { Counter } from 'k6/metrics'; + import { group, check, fail } from 'k6'; + + let customTags = { 'over': 'the rainbow' }; + let params = { 'tags': customTags}; + let statusCheck = { 'status is 200': (r) => r.status === 200 } + + let myCounter = new Counter('mycounter'); + + export const options = { + hosts: { + "HTTPBIN_DOMAIN": "HTTPBIN_IP", + "HTTPSBIN_DOMAIN": "HTTPSBIN_IP", + } + } + + export default function() { + group('http', function() { + check(http.get('HTTPSBIN_URL', params), statusCheck, customTags); + check(http.get('HTTPBIN_URL/status/418', params), statusCheck, customTags); + }) + + group('websockets', function() { + var response = ws.connect('WSBIN_URL/ws-echo', params, function (socket) { + socket.on('open', function open() { + console.log('ws open and say hello'); + socket.send('hello'); + }); + + socket.on('message', function (message) { + console.log('ws got message ' + message); + if (message != 'hello') { + fail('Expected to receive "hello" but got "' + message + '" instead !'); + } + console.log('ws closing socket...'); + socket.close(); + }); + + socket.on('close', function () { + console.log('ws close'); + }); + + socket.on('error', function (e) { + console.log('ws error: ' + e.error()); + }); + }); + console.log('connect returned'); + check(response, { 'status is 101': (r) => r && r.status === 101 }, customTags); + }) + + myCounter.add(1, customTags); + } + `)) + + ts := newGlobalTestState(t) + require.NoError(t, afero.WriteFile(ts.fs, filepath.Join(ts.cwd, "test.js"), script, 0o644)) + ts.args = []string{ + "k6", "run", "-u", "2", "--tag", "foo=bar", "--tag", "test=mest", "--tag", "over=written", + "--log-output=stdout", "--out", "json=results.json", "test.js", + } + ts.envVars = map[string]string{"K6_ITERATIONS": "3", "K6_INSECURE_SKIP_TLS_VERIFY": "true"} + newRootCommand(ts.globalState).execute() + + stdOut := ts.stdOut.String() + t.Log(stdOut) + + jsonResults, err := afero.ReadFile(ts.fs, "results.json") + require.NoError(t, err) + + expTags := map[string]string{"foo": "bar", "test": "mest", "over": "written", "scenario": "default"} + assert.Equal(t, float64(3), sum(getSampleValues(t, jsonResults, "iterations", expTags))) + assert.Less(t, float64(0), sum(getSampleValues(t, jsonResults, "iteration_duration", expTags))) + assert.Less(t, float64(0), sum(getSampleValues(t, jsonResults, "data_received", expTags))) + assert.Less(t, float64(0), sum(getSampleValues(t, jsonResults, "data_sent", expTags))) + + expTags["over"] = "the rainbow" // we overwrite this in most with custom tags in the script + assert.Equal(t, float64(6), sum(getSampleValues(t, jsonResults, "checks", expTags))) + assert.Equal(t, float64(3), sum(getSampleValues(t, jsonResults, "mycounter", expTags))) + + expTags["group"] = "::http" + assert.Equal(t, float64(3), sum(getSampleValues(t, jsonResults, "checks", expTags))) + assert.Equal(t, float64(6), sum(getSampleValues(t, jsonResults, "http_reqs", expTags))) + assert.Equal(t, 6, len(getSampleValues(t, jsonResults, "http_req_duration", expTags))) + expTags["expected_response"] = "true" + assert.Equal(t, float64(3), sum(getSampleValues(t, jsonResults, "http_reqs", expTags))) + assert.Equal(t, 3, len(getSampleValues(t, jsonResults, "http_req_duration", expTags))) + expTags["expected_response"] = "false" + assert.Equal(t, float64(3), sum(getSampleValues(t, jsonResults, "http_reqs", expTags))) + assert.Equal(t, 3, len(getSampleValues(t, jsonResults, "http_req_duration", expTags))) + delete(expTags, "expected_response") + + expTags["group"] = "::websockets" + assert.Equal(t, float64(3), sum(getSampleValues(t, jsonResults, "checks", expTags))) + assert.Equal(t, float64(3), sum(getSampleValues(t, jsonResults, "ws_sessions", expTags))) + assert.Equal(t, float64(3), sum(getSampleValues(t, jsonResults, "ws_msgs_sent", expTags))) + assert.Equal(t, float64(3), sum(getSampleValues(t, jsonResults, "ws_msgs_received", expTags))) + assert.Equal(t, 3, len(getSampleValues(t, jsonResults, "ws_session_duration", expTags))) + assert.Equal(t, 0, len(getSampleValues(t, jsonResults, "http_req_duration", expTags))) + expTags["check"] = "status is 101" + assert.Equal(t, float64(3), sum(getSampleValues(t, jsonResults, "checks", expTags))) +} diff --git a/core/engine_test.go b/core/engine_test.go index 9fce6be2613..4fe572d202b 100644 --- a/core/engine_test.go +++ b/core/engine_test.go @@ -720,6 +720,7 @@ func TestSentReceivedMetrics(t *testing.T) { }) } +// TODO: delete, functionality duplicated in cmd/integration_test.go func TestRunTags(t *testing.T) { t.Parallel() From 3e73e8783b13e7f82e3ce23a3c7b907f2e536e72 Mon Sep 17 00:00:00 2001 From: Nedyalko Andreev Date: Mon, 5 Dec 2022 09:57:40 +0200 Subject: [PATCH 9/9] Improve the code comments for obsolete Engine tests --- core/engine_test.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/core/engine_test.go b/core/engine_test.go index 4fe572d202b..07430c502e1 100644 --- a/core/engine_test.go +++ b/core/engine_test.go @@ -720,7 +720,8 @@ func TestSentReceivedMetrics(t *testing.T) { }) } -// TODO: delete, functionality duplicated in cmd/integration_test.go +// TODO: delete when implementing https://github.com/grafana/k6/issues/1889, the +// test functionality was duplicated in cmd/integration_test.go func TestRunTags(t *testing.T) { t.Parallel() @@ -841,7 +842,8 @@ func TestRunTags(t *testing.T) { } } -// TODO: delete, functionality duplicated in cmd/integration_test.go +// TODO: delete when implementing https://github.com/grafana/k6/issues/1889, the +// test functionality was duplicated in cmd/integration_test.go func TestSetupException(t *testing.T) { t.Parallel() @@ -894,7 +896,8 @@ func TestSetupException(t *testing.T) { } } -// TODO: delete, functionality duplicated in cmd/integration_test.go +// TODO: delete when implementing https://github.com/grafana/k6/issues/1889, the +// test functionality was duplicated in cmd/integration_test.go func TestVuInitException(t *testing.T) { t.Parallel() @@ -1111,7 +1114,8 @@ func TestMetricsEmission(t *testing.T) { } } -// TODO: delete, functionality duplicated in cmd/integration_test.go +// TODO: delete when implementing https://github.com/grafana/k6/issues/1889, the +// test functionality was duplicated in cmd/integration_test.go func TestMinIterationDurationInSetupTeardownStage(t *testing.T) { t.Parallel() setupScript := ` @@ -1192,7 +1196,8 @@ func TestMinIterationDurationInSetupTeardownStage(t *testing.T) { } } -// TODO: delete, functionality duplicated in cmd/integration_test.go +// TODO: delete when implementing https://github.com/grafana/k6/issues/1889, the +// test functionality was duplicated in cmd/integration_test.go func TestEngineRunsTeardownEvenAfterTestRunIsAborted(t *testing.T) { t.Parallel() @@ -1236,7 +1241,8 @@ func TestEngineRunsTeardownEvenAfterTestRunIsAborted(t *testing.T) { assert.Equal(t, 1.0, count) } -// TODO: delete, functionality duplicated in cmd/integration_test.go +// TODO: delete when implementing https://github.com/grafana/k6/issues/1889, the +// test functionality was duplicated in cmd/integration_test.go func TestActiveVUsCount(t *testing.T) { t.Parallel()