diff --git a/base/collect_grpc_test.go b/base/collect_grpc_test.go index ad77293db..f65d85ea9 100644 --- a/base/collect_grpc_test.go +++ b/base/collect_grpc_test.go @@ -342,3 +342,121 @@ func TestRunCollectGRPCSingleEndpointMultipleCalls(t *testing.T) { assert.NotNil(t, ghzResult[unary]) assert.NotNil(t, ghzResult[unary2]) } + +func TestRunCollectGRPCWithWarmup(t *testing.T) { + // define METRICS_SERVER_URL + metricsServerURL := "http://iter8.default:8080" + err := os.Setenv(MetricsServerURL, metricsServerURL) + assert.NoError(t, err) + + call := "helloworld.Greeter.SayHello" + + _ = os.Chdir(t.TempDir()) + callType := helloworld.Unary + gs, s, err := internal.StartServer(false) + if err != nil { + assert.FailNow(t, err.Error()) + } + t.Cleanup(s.Stop) + + // valid collect GRPC task... should succeed + warmupTrue := true + ct := &collectGRPCTask{ + TaskMeta: TaskMeta{ + Task: StringPointer(CollectGRPCTaskName), + }, + With: collectGRPCInputs{ + Config: runner.Config{ + Data: map[string]interface{}{"name": "bob"}, + Call: call, + Host: internal.LocalHostPort, + }, + Warmup: &warmupTrue, + }, + } + + log.Logger.Debug("dial timeout before defaulting... ", ct.With.DialTimeout.String()) + + exp := &Experiment{ + Spec: []Task{ct}, + Result: &ExperimentResult{}, + Metadata: ExperimentMetadata{ + Name: myName, + Namespace: myNamespace, + }, + } + exp.initResults(1) + err = ct.run(exp) + + log.Logger.Debug("dial timeout after defaulting... ", ct.With.DialTimeout.String()) + + assert.NoError(t, err) + + count := gs.GetCount(callType) + assert.Equal(t, 200, count) + + // warmup option ensures that ghz results are not written to insights + assert.Nil(t, exp.Result.Insights) +} + +// Credit: Several of the tests in this file are based on +// https://github.com/bojand/ghz/blob/master/runner/run_test.go +func TestRunCollectGRPCWithIncorrectNumVersions(t *testing.T) { + // define METRICS_SERVER_URL + metricsServerURL := "http://iter8.default:8080" + err := os.Setenv(MetricsServerURL, metricsServerURL) + assert.NoError(t, err) + + call := "helloworld.Greeter.SayHello" + + _ = os.Chdir(t.TempDir()) + callType := helloworld.Unary + gs, s, err := internal.StartServer(false) + if err != nil { + assert.FailNow(t, err.Error()) + } + t.Cleanup(s.Stop) + + // valid collect GRPC task... should succeed + ct := &collectGRPCTask{ + TaskMeta: TaskMeta{ + Task: StringPointer(CollectGRPCTaskName), + }, + With: collectGRPCInputs{ + Config: runner.Config{ + Data: map[string]interface{}{"name": "bob"}, + Call: call, + Host: internal.LocalHostPort, + }, + }, + } + + log.Logger.Debug("dial timeout before defaulting... ", ct.With.DialTimeout.String()) + + exp := &Experiment{ + Spec: []Task{ct}, + Result: &ExperimentResult{}, + Metadata: ExperimentMetadata{ + Name: myName, + Namespace: myNamespace, + }, + } + exp.initResults(1) + + exp.Result.Insights = &Insights{ + NumVersions: 2, // will cause grpc task to fail; grpc task expects insights been nil or numVersions set to 1 + } + + err = ct.run(exp) + + log.Logger.Debug("dial timeout after defaulting... ", ct.With.DialTimeout.String()) + + // fail because of initInsightsWithNumVersions() + assert.Error(t, err) + + count := gs.GetCount(callType) + assert.Equal(t, 200, count) + + // error ensures that ghz results are not written to insights + assert.Nil(t, exp.Result.Insights.TaskData) +} diff --git a/base/collect_http.go b/base/collect_http.go index 9c50525fd..f1679e1b7 100644 --- a/base/collect_http.go +++ b/base/collect_http.go @@ -46,14 +46,15 @@ type endpoint struct { URL string `json:"url" yaml:"url"` // AllowInitialErrors allows and doesn't abort on initial warmup errors AllowInitialErrors *bool `json:"allowInitialErrors,omitempty" yaml:"allowInitialErrors,omitempty"` - // Warmup indicates if task execution is for warmup purposes; if so the results will be ignored - Warmup *bool `json:"warmup,omitempty" yaml:"warmup,omitempty"` } // collectHTTPInputs contain the inputs to the metrics collection task to be executed. type collectHTTPInputs struct { endpoint + // Warmup indicates if task execution is for warmup purposes; if so the results will be ignored + Warmup *bool `json:"warmup,omitempty" yaml:"warmup,omitempty"` + // Endpoints is used to define multiple endpoints to test Endpoints map[string]endpoint `json:"endpoints" yaml:"endpoints"` } @@ -81,30 +82,6 @@ var ( defaultPercentiles = [...]float64{50.0, 75.0, 90.0, 95.0, 99.0, 99.9} ) -// errorCode checks if a given code is an error code -func (t *collectHTTPTask) errorCode(code int) bool { - // connection failure - if code == -1 { - return true - } - // HTTP errors - for _, lims := range t.With.ErrorRanges { - // if no lower limit (check upper) - if lims.Lower == nil && code <= *lims.Upper { - return true - } - // if no upper limit (check lower) - if lims.Upper == nil && code >= *lims.Lower { - return true - } - // if both limits are present (check both) - if lims.Upper != nil && lims.Lower != nil && code <= *lims.Upper && code >= *lims.Lower { - return true - } - } - return false -} - // collectHTTPTask enables performance testing of HTTP services. type collectHTTPTask struct { // TaskMeta has fields common to all tasks diff --git a/base/collect_http_test.go b/base/collect_http_test.go index 1c2900809..b82762933 100644 --- a/base/collect_http_test.go +++ b/base/collect_http_test.go @@ -370,30 +370,122 @@ func TestRunCollectHTTPMultipleNoEndpoints(t *testing.T) { assert.Equal(t, 0, len(httpResult)) } -func TestErrorCode(t *testing.T) { - task := collectHTTPTask{} - assert.True(t, task.errorCode(-1)) - - // if no lower limit (check upper) - upper := 10 - task.With.ErrorRanges = append(task.With.ErrorRanges, errorRange{ - Upper: &upper, - }) - assert.True(t, task.errorCode(5)) - - // if no upper limit (check lower) - task.With.ErrorRanges = []errorRange{} - lower := 1 - task.With.ErrorRanges = append(task.With.ErrorRanges, errorRange{ - Lower: &lower, - }) - assert.True(t, task.errorCode(5)) - - // if both limits are present (check both) - task.With.ErrorRanges = []errorRange{} - task.With.ErrorRanges = append(task.With.ErrorRanges, errorRange{ - Upper: &upper, - Lower: &lower, - }) - assert.True(t, task.errorCode(5)) +func TestRunCollectHTTPWithWarmup(t *testing.T) { + // define METRICS_SERVER_URL + metricsServerURL := "http://iter8.default:8080" + err := os.Setenv(MetricsServerURL, metricsServerURL) + assert.NoError(t, err) + + mux, addr := fhttp.DynamicHTTPServer(false) + + // /foo/ handler + called := false // ensure that the /foo/ handler is called + handler := func(w http.ResponseWriter, r *http.Request) { + called = true + data, _ := io.ReadAll(r.Body) + testData, _ := os.ReadFile(CompletePath("../", "testdata/payload/ukpolice.json")) + + // assert that PayloadFile is working + assert.True(t, bytes.Equal(data, testData)) + + w.WriteHeader(200) + } + mux.HandleFunc("/"+foo, handler) + + url := fmt.Sprintf("http://localhost:%d/", addr.Port) + foo + + // valid collect HTTP task... should succeed + warmupTrue := true + ct := &collectHTTPTask{ + TaskMeta: TaskMeta{ + Task: StringPointer(CollectHTTPTaskName), + }, + With: collectHTTPInputs{ + endpoint: endpoint{ + Duration: StringPointer("1s"), + PayloadFile: StringPointer(CompletePath("../", "testdata/payload/ukpolice.json")), + Headers: map[string]string{}, + URL: url, + }, + Warmup: &warmupTrue, + }, + } + + exp := &Experiment{ + Spec: []Task{ct}, + Result: &ExperimentResult{}, + Metadata: ExperimentMetadata{ + Name: myName, + Namespace: myNamespace, + }, + } + exp.initResults(1) + err = ct.run(exp) + assert.NoError(t, err) + assert.True(t, called) // ensure that the /foo/ handler is called + + // warmup option ensures that Fortio results are not written to insights + assert.Nil(t, exp.Result.Insights) +} + +func TestRunCollectHTTPWithIncorrectNumVersions(t *testing.T) { + // define METRICS_SERVER_URL + metricsServerURL := "http://iter8.default:8080" + err := os.Setenv(MetricsServerURL, metricsServerURL) + assert.NoError(t, err) + + mux, addr := fhttp.DynamicHTTPServer(false) + + // /foo/ handler + called := false // ensure that the /foo/ handler is called + handler := func(w http.ResponseWriter, r *http.Request) { + called = true + data, _ := io.ReadAll(r.Body) + testData, _ := os.ReadFile(CompletePath("../", "testdata/payload/ukpolice.json")) + + // assert that PayloadFile is working + assert.True(t, bytes.Equal(data, testData)) + + w.WriteHeader(200) + } + mux.HandleFunc("/"+foo, handler) + + url := fmt.Sprintf("http://localhost:%d/", addr.Port) + foo + + // valid collect HTTP task... should succeed + ct := &collectHTTPTask{ + TaskMeta: TaskMeta{ + Task: StringPointer(CollectHTTPTaskName), + }, + With: collectHTTPInputs{ + endpoint: endpoint{ + Duration: StringPointer("1s"), + PayloadFile: StringPointer(CompletePath("../", "testdata/payload/ukpolice.json")), + Headers: map[string]string{}, + URL: url, + }, + }, + } + + exp := &Experiment{ + Spec: []Task{ct}, + Result: &ExperimentResult{}, + Metadata: ExperimentMetadata{ + Name: myName, + Namespace: myNamespace, + }, + } + exp.initResults(1) + + exp.Result.Insights = &Insights{ + NumVersions: 2, // will cause http task to fail; grpc task expects insights been nil or numVersions set to 1 + } + + err = ct.run(exp) + assert.Error(t, err) // fail because of initInsightsWithNumVersions() + + assert.True(t, called) // ensure that the /foo/ handler is called + + // error ensures that Fortio results are not written to insights + assert.Nil(t, exp.Result.Insights.TaskData) } diff --git a/base/experiment_test.go b/base/experiment_test.go index 2c3b94d69..8be431d06 100644 --- a/base/experiment_test.go +++ b/base/experiment_test.go @@ -145,6 +145,32 @@ func TestFailExperiment(t *testing.T) { assert.False(t, exp.NoFailure()) } +func TestUnmarshalJSON(t *testing.T) { + tests := []struct { + specBytes string + errMessage string + }{ + { + specBytes: `[{"task":"ready"}]`, + }, + { + specBytes: `[{"task":"http"}]`, + }, + { + specBytes: `[{"task":"grpc"}]`, + }, + { + specBytes: `[{"task":"notify"}]`, + }, + } + + for _, test := range tests { + exp := ExperimentSpec{} + err := exp.UnmarshalJSON([]byte(test.specBytes)) + assert.NoError(t, err) + } +} + func TestUnmarshalJSONError(t *testing.T) { tests := []struct { specBytes string @@ -171,3 +197,18 @@ func TestUnmarshalJSONError(t *testing.T) { assert.EqualError(t, err, test.errMessage) } } + +func TestInitInsightsWithNumVersions(t *testing.T) { + r := ExperimentResult{ + Insights: &Insights{ + NumVersions: 1, + }, + } + + err := r.initInsightsWithNumVersions(1) + assert.NoError(t, err) + + // Mismatching version numbers + err = r.initInsightsWithNumVersions(2) + assert.Error(t, err) +} diff --git a/cmd/version.go b/cmd/version.go index e4988255d..24390c450 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -12,7 +12,7 @@ import ( var versionDesc = ` Print the version of Iter8 CLI. - iter8 version + $ iter8 version The output may look as follows: