Add tests

Signed-off-by: Alan Cha <Alan.cha1@ibm.com>
iter8-tools · Sep 6, 2023 · d64a5fc · d64a5fc
1 parent ab4c467
commit d64a5fc
Show file tree

Hide file tree

Showing 5 changed files with 281 additions and 53 deletions.
diff --git a/base/collect_grpc_test.go b/base/collect_grpc_test.go
@@ -342,3 +342,121 @@ func TestRunCollectGRPCSingleEndpointMultipleCalls(t *testing.T) {
 	assert.NotNil(t, ghzResult[unary])
 	assert.NotNil(t, ghzResult[unary2])
 }
+
+func TestRunCollectGRPCWithWarmup(t *testing.T) {
+	// define METRICS_SERVER_URL
+	metricsServerURL := "http://iter8.default:8080"
+	err := os.Setenv(MetricsServerURL, metricsServerURL)
+	assert.NoError(t, err)
+
+	call := "helloworld.Greeter.SayHello"
+
+	_ = os.Chdir(t.TempDir())
+	callType := helloworld.Unary
+	gs, s, err := internal.StartServer(false)
+	if err != nil {
+		assert.FailNow(t, err.Error())
+	}
+	t.Cleanup(s.Stop)
+
+	// valid collect GRPC task... should succeed
+	warmupTrue := true
+	ct := &collectGRPCTask{
+		TaskMeta: TaskMeta{
+			Task: StringPointer(CollectGRPCTaskName),
+		},
+		With: collectGRPCInputs{
+			Config: runner.Config{
+				Data: map[string]interface{}{"name": "bob"},
+				Call: call,
+				Host: internal.LocalHostPort,
+			},
+			Warmup: &warmupTrue,
+		},
+	}
+
+	log.Logger.Debug("dial timeout before defaulting... ", ct.With.DialTimeout.String())
+
+	exp := &Experiment{
+		Spec:   []Task{ct},
+		Result: &ExperimentResult{},
+		Metadata: ExperimentMetadata{
+			Name:      myName,
+			Namespace: myNamespace,
+		},
+	}
+	exp.initResults(1)
+	err = ct.run(exp)
+
+	log.Logger.Debug("dial timeout after defaulting... ", ct.With.DialTimeout.String())
+
+	assert.NoError(t, err)
+
+	count := gs.GetCount(callType)
+	assert.Equal(t, 200, count)
+
+	// warmup option ensures that ghz results are not written to insights
+	assert.Nil(t, exp.Result.Insights)
+}
+
+// Credit: Several of the tests in this file are based on
+// https://github.com/bojand/ghz/blob/master/runner/run_test.go
+func TestRunCollectGRPCWithIncorrectNumVersions(t *testing.T) {
+	// define METRICS_SERVER_URL
+	metricsServerURL := "http://iter8.default:8080"
+	err := os.Setenv(MetricsServerURL, metricsServerURL)
+	assert.NoError(t, err)
+
+	call := "helloworld.Greeter.SayHello"
+
+	_ = os.Chdir(t.TempDir())
+	callType := helloworld.Unary
+	gs, s, err := internal.StartServer(false)
+	if err != nil {
+		assert.FailNow(t, err.Error())
+	}
+	t.Cleanup(s.Stop)
+
+	// valid collect GRPC task... should succeed
+	ct := &collectGRPCTask{
+		TaskMeta: TaskMeta{
+			Task: StringPointer(CollectGRPCTaskName),
+		},
+		With: collectGRPCInputs{
+			Config: runner.Config{
+				Data: map[string]interface{}{"name": "bob"},
+				Call: call,
+				Host: internal.LocalHostPort,
+			},
+		},
+	}
+
+	log.Logger.Debug("dial timeout before defaulting... ", ct.With.DialTimeout.String())
+
+	exp := &Experiment{
+		Spec:   []Task{ct},
+		Result: &ExperimentResult{},
+		Metadata: ExperimentMetadata{
+			Name:      myName,
+			Namespace: myNamespace,
+		},
+	}
+	exp.initResults(1)
+
+	exp.Result.Insights = &Insights{
+		NumVersions: 2, // will cause grpc task to fail; grpc task expects insights been nil or numVersions set to 1
+	}
+
+	err = ct.run(exp)
+
+	log.Logger.Debug("dial timeout after defaulting... ", ct.With.DialTimeout.String())
+
+	// fail because of initInsightsWithNumVersions()
+	assert.Error(t, err)
+
+	count := gs.GetCount(callType)
+	assert.Equal(t, 200, count)
+
+	// error ensures that ghz results are not written to insights
+	assert.Nil(t, exp.Result.Insights.TaskData)
+}
diff --git a/base/collect_http.go b/base/collect_http.go
@@ -46,14 +46,15 @@ type endpoint struct {
 	URL string `json:"url" yaml:"url"`
 	// AllowInitialErrors allows and doesn't abort on initial warmup errors
 	AllowInitialErrors *bool `json:"allowInitialErrors,omitempty" yaml:"allowInitialErrors,omitempty"`
-	// Warmup indicates if task execution is for warmup purposes; if so the results will be ignored
-	Warmup *bool `json:"warmup,omitempty" yaml:"warmup,omitempty"`
 }
 
 // collectHTTPInputs contain the inputs to the metrics collection task to be executed.
 type collectHTTPInputs struct {
 	endpoint
 
+	// Warmup indicates if task execution is for warmup purposes; if so the results will be ignored
+	Warmup *bool `json:"warmup,omitempty" yaml:"warmup,omitempty"`
+
 	// Endpoints is used to define multiple endpoints to test
 	Endpoints map[string]endpoint `json:"endpoints" yaml:"endpoints"`
 }
@@ -81,30 +82,6 @@ var (
 	defaultPercentiles = [...]float64{50.0, 75.0, 90.0, 95.0, 99.0, 99.9}
 )
 
-// errorCode checks if a given code is an error code
-func (t *collectHTTPTask) errorCode(code int) bool {
-	// connection failure
-	if code == -1 {
-		return true
-	}
-	// HTTP errors
-	for _, lims := range t.With.ErrorRanges {
-		// if no lower limit (check upper)
-		if lims.Lower == nil && code <= *lims.Upper {
-			return true
-		}
-		// if no upper limit (check lower)
-		if lims.Upper == nil && code >= *lims.Lower {
-			return true
-		}
-		// if both limits are present (check both)
-		if lims.Upper != nil && lims.Lower != nil && code <= *lims.Upper && code >= *lims.Lower {
-			return true
-		}
-	}
-	return false
-}
-
 // collectHTTPTask enables performance testing of HTTP services.
 type collectHTTPTask struct {
 	// TaskMeta has fields common to all tasks

diff --git a/base/collect_http_test.go b/base/collect_http_test.go
@@ -370,30 +370,122 @@ func TestRunCollectHTTPMultipleNoEndpoints(t *testing.T) {
 	assert.Equal(t, 0, len(httpResult))
 }
 
-func TestErrorCode(t *testing.T) {
-	task := collectHTTPTask{}
-	assert.True(t, task.errorCode(-1))
-
-	// if no lower limit (check upper)
-	upper := 10
-	task.With.ErrorRanges = append(task.With.ErrorRanges, errorRange{
-		Upper: &upper,
-	})
-	assert.True(t, task.errorCode(5))
-
-	// if no upper limit (check lower)
-	task.With.ErrorRanges = []errorRange{}
-	lower := 1
-	task.With.ErrorRanges = append(task.With.ErrorRanges, errorRange{
-		Lower: &lower,
-	})
-	assert.True(t, task.errorCode(5))
-
-	// if both limits are present (check both)
-	task.With.ErrorRanges = []errorRange{}
-	task.With.ErrorRanges = append(task.With.ErrorRanges, errorRange{
-		Upper: &upper,
-		Lower: &lower,
-	})
-	assert.True(t, task.errorCode(5))
+func TestRunCollectHTTPWithWarmup(t *testing.T) {
+	// define METRICS_SERVER_URL
+	metricsServerURL := "http://iter8.default:8080"
+	err := os.Setenv(MetricsServerURL, metricsServerURL)
+	assert.NoError(t, err)
+
+	mux, addr := fhttp.DynamicHTTPServer(false)
+
+	// /foo/ handler
+	called := false // ensure that the /foo/ handler is called
+	handler := func(w http.ResponseWriter, r *http.Request) {
+		called = true
+		data, _ := io.ReadAll(r.Body)
+		testData, _ := os.ReadFile(CompletePath("../", "testdata/payload/ukpolice.json"))
+
+		// assert that PayloadFile is working
+		assert.True(t, bytes.Equal(data, testData))
+
+		w.WriteHeader(200)
+	}
+	mux.HandleFunc("/"+foo, handler)
+
+	url := fmt.Sprintf("http://localhost:%d/", addr.Port) + foo
+
+	// valid collect HTTP task... should succeed
+	warmupTrue := true
+	ct := &collectHTTPTask{
+		TaskMeta: TaskMeta{
+			Task: StringPointer(CollectHTTPTaskName),
+		},
+		With: collectHTTPInputs{
+			endpoint: endpoint{
+				Duration:    StringPointer("1s"),
+				PayloadFile: StringPointer(CompletePath("../", "testdata/payload/ukpolice.json")),
+				Headers:     map[string]string{},
+				URL:         url,
+			},
+			Warmup: &warmupTrue,
+		},
+	}
+
+	exp := &Experiment{
+		Spec:   []Task{ct},
+		Result: &ExperimentResult{},
+		Metadata: ExperimentMetadata{
+			Name:      myName,
+			Namespace: myNamespace,
+		},
+	}
+	exp.initResults(1)
+	err = ct.run(exp)
+	assert.NoError(t, err)
+	assert.True(t, called) // ensure that the /foo/ handler is called
+
+	// warmup option ensures that Fortio results are not written to insights
+	assert.Nil(t, exp.Result.Insights)
+}
+
+func TestRunCollectHTTPWithIncorrectNumVersions(t *testing.T) {
+	// define METRICS_SERVER_URL
+	metricsServerURL := "http://iter8.default:8080"
+	err := os.Setenv(MetricsServerURL, metricsServerURL)
+	assert.NoError(t, err)
+
+	mux, addr := fhttp.DynamicHTTPServer(false)
+
+	// /foo/ handler
+	called := false // ensure that the /foo/ handler is called
+	handler := func(w http.ResponseWriter, r *http.Request) {
+		called = true
+		data, _ := io.ReadAll(r.Body)
+		testData, _ := os.ReadFile(CompletePath("../", "testdata/payload/ukpolice.json"))
+
+		// assert that PayloadFile is working
+		assert.True(t, bytes.Equal(data, testData))
+
+		w.WriteHeader(200)
+	}
+	mux.HandleFunc("/"+foo, handler)
+
+	url := fmt.Sprintf("http://localhost:%d/", addr.Port) + foo
+
+	// valid collect HTTP task... should succeed
+	ct := &collectHTTPTask{
+		TaskMeta: TaskMeta{
+			Task: StringPointer(CollectHTTPTaskName),
+		},
+		With: collectHTTPInputs{
+			endpoint: endpoint{
+				Duration:    StringPointer("1s"),
+				PayloadFile: StringPointer(CompletePath("../", "testdata/payload/ukpolice.json")),
+				Headers:     map[string]string{},
+				URL:         url,
+			},
+		},
+	}
+
+	exp := &Experiment{
+		Spec:   []Task{ct},
+		Result: &ExperimentResult{},
+		Metadata: ExperimentMetadata{
+			Name:      myName,
+			Namespace: myNamespace,
+		},
+	}
+	exp.initResults(1)
+
+	exp.Result.Insights = &Insights{
+		NumVersions: 2, // will cause http task to fail; grpc task expects insights been nil or numVersions set to 1
+	}
+
+	err = ct.run(exp)
+	assert.Error(t, err) // fail because of initInsightsWithNumVersions()
+
+	assert.True(t, called) // ensure that the /foo/ handler is called
+
+	// error ensures that Fortio results are not written to insights
+	assert.Nil(t, exp.Result.Insights.TaskData)
 }
diff --git a/base/experiment_test.go b/base/experiment_test.go
@@ -145,6 +145,32 @@ func TestFailExperiment(t *testing.T) {
 	assert.False(t, exp.NoFailure())
 }
 
+func TestUnmarshalJSON(t *testing.T) {
+	tests := []struct {
+		specBytes  string
+		errMessage string
+	}{
+		{
+			specBytes: `[{"task":"ready"}]`,
+		},
+		{
+			specBytes: `[{"task":"http"}]`,
+		},
+		{
+			specBytes: `[{"task":"grpc"}]`,
+		},
+		{
+			specBytes: `[{"task":"notify"}]`,
+		},
+	}
+
+	for _, test := range tests {
+		exp := ExperimentSpec{}
+		err := exp.UnmarshalJSON([]byte(test.specBytes))
+		assert.NoError(t, err)
+	}
+}
+
 func TestUnmarshalJSONError(t *testing.T) {
 	tests := []struct {
 		specBytes  string
@@ -171,3 +197,18 @@ func TestUnmarshalJSONError(t *testing.T) {
 		assert.EqualError(t, err, test.errMessage)
 	}
 }
+
+func TestInitInsightsWithNumVersions(t *testing.T) {
+	r := ExperimentResult{
+		Insights: &Insights{
+			NumVersions: 1,
+		},
+	}
+
+	err := r.initInsightsWithNumVersions(1)
+	assert.NoError(t, err)
+
+	// Mismatching version numbers
+	err = r.initInsightsWithNumVersions(2)
+	assert.Error(t, err)
+}
diff --git a/cmd/version.go b/cmd/version.go
@@ -12,7 +12,7 @@ import (
 var versionDesc = `
 Print the version of Iter8 CLI.
 
-	iter8 version
+	$ iter8 version
 
 The output may look as follows: