feat!: replace StopMagic with autocompleteHeaders

README.md

@@ -321,7 +321,7 @@ Tests can be altered using four lists:
   - `headers`: overrides headers, the format is a map of strings
   - `method`: overrides the method used to perform the request
   - `data`: overrides data sent in the request
-  - `stopmagic`: prevent header autocompletion (currently sets `Connection: close` and `Content-Length` for requests with body data)
+  - `autocomplete_headers`: prevent header autocompletion (currently sets `Connection: close` and `Content-Length` for requests with body data)
   - `encodedrequest`: overrides base64 encoded request
   - `rawrequest`: permits to provide a raw request. `method`, `uri` and `version` values will be ignored
 - `ignore` is for tests you want to ignore. You should add a comment on why you ignore the test

check/base.go

@@ -47,7 +47,7 @@ func (c *FTWCheck) SetExpectResponse(response string) {
 
 // SetExpectError sets the boolean if we are expecting an error from the server
 func (c *FTWCheck) SetExpectError(expect bool) {
-	c.expected.ExpectError = expect
+	c.expected.ExpectError = &expect
 }
 
 // SetLogContains sets the string to look for in logs

check/base_test.go

@@ -62,11 +62,11 @@ func (s *checkBaseTestSuite) TestNewCheck() {
 		ResponseContains: "",
 		LogContains:      "nothing",
 		NoLogContains:    "",
-		ExpectError:      true,
+		ExpectError:      func() *bool { b := true; return &b }(),
 	}
 	c.SetExpectTestOutput(&to)
 
-	s.True(c.expected.ExpectError, "Problem setting expected output")
+	s.True(*c.expected.ExpectError, "Problem setting expected output")
 
 	c.SetNoLogContains("nologcontains")
 

check/error.go

@@ -5,11 +5,11 @@ import "github.com/rs/zerolog/log"
 // AssertExpectError helper to check if this error was expected or not
 func (c *FTWCheck) AssertExpectError(err error) bool {
 	if err != nil {
-		log.Debug().Msgf("ftw/check: expected error? -> %t, and error is %s", c.expected.ExpectError, err.Error())
+		log.Debug().Msgf("ftw/check: expected error? -> %t, and error is %s", *c.expected.ExpectError, err.Error())
 	} else {
-		log.Debug().Msgf("ftw/check: expected error? -> %t, and error is nil", c.expected.ExpectError)
+		log.Debug().Msgf("ftw/check: expected error? -> %t, and error is nil", *c.expected.ExpectError)
 	}
-	if c.expected.ExpectError && err != nil {
+	if *c.expected.ExpectError && err != nil {
 		return true
 	}
 	return false

cmd/root.go

@@ -68,4 +68,5 @@ func initConfig() {
 	if cloud {
 		cfg.RunMode = config.CloudRunMode
 	}
+
 }

config/config.go

@@ -35,17 +35,12 @@ func NewCloudConfig() *FTWConfiguration {
 // NewConfigFromFile reads configuration information from the config file if it exists,
 // or uses `.ftw.yaml` as default file
 func NewConfigFromFile(cfgFile string) (*FTWConfiguration, error) {
-	// Global koanf instance. Use "." as the key path delimiter. This can be "/" or any character.
-	var k = koanf.New(".")
-	var err error
-	cfg := NewDefaultConfig()
-
 	// first check if we had an explicit call with config file
 	if cfgFile == "" {
 		cfgFile = ".ftw.yaml"
 	}
 
-	_, err = os.Stat(cfgFile)
+	_, err := os.Stat(cfgFile)
 	if err != nil { // file does not exist, so we try the home folder
 
 		var home string
@@ -64,56 +59,39 @@ func NewConfigFromFile(cfgFile string) (*FTWConfiguration, error) {
 		return nil, err
 	}
 
+	k := getKoanfInstance()
 	err = k.Load(file.Provider(cfgFile), yaml.Parser())
 	if err != nil {
 		return nil, err
 	}
 
-	// At this point we have loaded our config, now we need to
-	// unmarshal the whole root module
-	err = k.UnmarshalWithConf("", cfg, koanf.UnmarshalConf{Tag: "koanf"})
-	if err != nil {
-		return nil, err
-	}
-
-	return cfg, err
+	return unmarshal(k)
 }
 
 // NewConfigFromEnv reads configuration information from environment variables that start with `FTW_`
 func NewConfigFromEnv() (*FTWConfiguration, error) {
-	var err error
-	var k = koanf.New(".")
-	cfg := NewDefaultConfig()
-
-	err = k.Load(env.Provider("FTW_", ".", func(s string) string {
+	k := getKoanfInstance()
+	err := k.Load(env.Provider("FTW_", ".", func(s string) string {
 		return strings.ReplaceAll(strings.ToLower(
 			strings.TrimPrefix(s, "FTW_")), "_", ".")
 	}), nil)
 
 	if err != nil {
 		return nil, err
 	}
-	// Unmarshal the whole root module
-	err = k.UnmarshalWithConf("", cfg, koanf.UnmarshalConf{Tag: "koanf"})
 
-	return cfg, err
+	return unmarshal(k)
 }
 
 // NewConfigFromString initializes the configuration from a yaml formatted string. Useful for testing.
 func NewConfigFromString(conf string) (*FTWConfiguration, error) {
-	var k = koanf.New(".")
-	var err error
-	cfg := NewDefaultConfig()
-
-	err = k.Load(rawbytes.Provider([]byte(conf)), yaml.Parser())
+	k := getKoanfInstance()
+	err := k.Load(rawbytes.Provider([]byte(conf)), yaml.Parser())
 	if err != nil {
 		return nil, err
 	}
 
-	// Unmarshal the whole root module
-	err = k.UnmarshalWithConf("", cfg, koanf.UnmarshalConf{Tag: "koanf"})
-
-	return cfg, err
+	return unmarshal(k)
 }
 
 // WithLogfile changes the logfile in the configuration.
@@ -145,3 +123,20 @@ func (c *FTWConfiguration) WithMaxMarkerRetries(retries int) {
 func (c *FTWConfiguration) WithMaxMarkerLogLines(amount int) {
 	c.MaxMarkerLogLines = amount
 }
+
+// Unmarshal the loaded koanf instance into a configuration object
+func unmarshal(k *koanf.Koanf) (*FTWConfiguration, error) {
+	config := NewDefaultConfig()
+	err := k.UnmarshalWithConf("", config, koanf.UnmarshalConf{Tag: "koanf"})
+	if err != nil {
+		return nil, err
+	}
+
+	return config, nil
+}
+
+// Get the global koanf instance
+func getKoanfInstance() *koanf.Koanf {
+	// Use "." as the key path delimiter. This can be "/" or any character.
+	return koanf.New(".")
+}

ftwhttp/types.go

@@ -62,7 +62,7 @@ type RequestLine struct {
 }
 
 // Request represents a request
-// No Defaults represents the previous "stop_magic" behavior
+// This struct without defaults represents the previous "autocomplete headers" behavior
 type Request struct {
 	requestLine         *RequestLine
 	headers             Header

runner/run.go

@@ -19,7 +19,7 @@ import (
 	"github.com/coreruleset/go-ftw/waflog"
 )
 
-var errBadTestRequest = errors.New("ftw/run: bad test: choose between data, encoded_request, or raw_request")
+var errBadTestInput = errors.New("ftw/run: bad test input: choose between data, encoded_request, or raw_request")
 
 // Run runs your tests with the specified Config.
 func Run(cfg *config.FTWConfiguration, tests []test.FTWTest, c RunnerConfig, out *output.Output) (*TestRunContext, error) {
@@ -75,9 +75,9 @@ func RunTest(runContext *TestRunContext, ftwTest test.FTWTest) error {
 
 	for _, testCase := range ftwTest.Tests {
 		// if we received a particular testid, skip until we find it
-		if needToSkipTest(runContext.Include, runContext.Exclude, testCase.TestTitle, ftwTest.Meta.Enabled) {
+		if needToSkipTest(runContext.Include, runContext.Exclude, testCase.TestTitle, *ftwTest.Meta.Enabled) {
 			runContext.Stats.addResultToStats(Skipped, testCase.TestTitle, 0)
-			if !ftwTest.Meta.Enabled && !runContext.ShowOnlyFailed {
+			if !*ftwTest.Meta.Enabled && !runContext.ShowOnlyFailed {
 				runContext.Output.Println("\tskipping %s - (enabled: false) in file.", testCase.TestTitle)
 			}
 			continue
@@ -115,16 +115,13 @@ func RunStage(runContext *TestRunContext, ftwCheck *check.FTWCheck, testCase tes
 	stageStartTime := time.Now()
 	stageID := uuid.NewString()
 	// Apply global overrides initially
-	testRequest := stage.Input
-	err := applyInputOverride(runContext.Config.TestOverride, &testRequest)
-	if err != nil {
-		log.Debug().Msgf("ftw/run: problem overriding input: %s", err.Error())
-	}
+	testInput := stage.Input
+	test.ApplyInputOverrides(&runContext.Config.TestOverride.Overrides, &testInput)
 	expectedOutput := stage.Output
 
 	// Check sanity first
-	if checkTestSanity(testRequest) {
-		return errBadTestRequest
+	if checkTestSanity(testInput) {
+		return errBadTestInput
 	}
 
 	// Do not even run test if result is overridden. Just use the override and display the overridden result.
@@ -138,38 +135,38 @@ func RunStage(runContext *TestRunContext, ftwCheck *check.FTWCheck, testCase tes
 
 	// Destination is needed for a request
 	dest := &ftwhttp.Destination{
-		DestAddr: testRequest.GetDestAddr(),
-		Port:     testRequest.GetPort(),
-		Protocol: testRequest.GetProtocol(),
+		DestAddr: testInput.GetDestAddr(),
+		Port:     testInput.GetPort(),
+		Protocol: testInput.GetProtocol(),
 	}
 
 	if notRunningInCloudMode(ftwCheck) {
 		startMarker, err := markAndFlush(runContext, dest, stageID)
-		if err != nil && !expectedOutput.ExpectError {
+		if err != nil && !*expectedOutput.ExpectError {
 			return fmt.Errorf("failed to find start marker: %w", err)
 		}
 		ftwCheck.SetStartMarker(startMarker)
 	}
 
-	req = getRequestFromTest(testRequest)
+	req = getRequestFromTest(testInput)
 
-	err = runContext.Client.NewConnection(*dest)
+	err := runContext.Client.NewConnection(*dest)
 
-	if err != nil && !expectedOutput.ExpectError {
+	if err != nil && !*expectedOutput.ExpectError {
 		return fmt.Errorf("can't connect to destination %+v: %w", dest, err)
 	}
 	runContext.Client.StartTrackingTime()
 
 	response, responseErr := runContext.Client.Do(*req)
 
 	runContext.Client.StopTrackingTime()
-	if responseErr != nil && !expectedOutput.ExpectError {
+	if responseErr != nil && !*expectedOutput.ExpectError {
 		return fmt.Errorf("failed sending request to destination %+v: %w", dest, responseErr)
 	}
 
 	if notRunningInCloudMode(ftwCheck) {
 		endMarker, err := markAndFlush(runContext, dest, stageID)
-		if err != nil && !expectedOutput.ExpectError {
+		if err != nil && !*expectedOutput.ExpectError {
 			return fmt.Errorf("failed to find end marker: %w", err)
 
 		}
@@ -270,10 +267,10 @@ func needToSkipTest(include *regexp.Regexp, exclude *regexp.Regexp, title string
 	return result
 }
 
-func checkTestSanity(testRequest test.Input) bool {
-	return (utils.IsNotEmpty(testRequest.Data) && testRequest.EncodedRequest != "") ||
-		(utils.IsNotEmpty(testRequest.Data) && testRequest.RAWRequest != "") ||
-		(testRequest.EncodedRequest != "" && testRequest.RAWRequest != "")
+func checkTestSanity(testInput test.Input) bool {
+	return (utils.IsNotEmpty(testInput.Data) && testInput.EncodedRequest != "") ||
+		(utils.IsNotEmpty(testInput.Data) && testInput.RAWRequest != "") ||
+		(testInput.EncodedRequest != "" && testInput.RAWRequest != "")
 }
 
 func displayResult(rc *TestRunContext, result TestResult, roundTripTime time.Duration, stageTime time.Duration) {
@@ -353,95 +350,33 @@ func checkResult(c *check.FTWCheck, response *ftwhttp.Response, responseError er
 	return Success
 }
 
-func getRequestFromTest(testRequest test.Input) *ftwhttp.Request {
+func getRequestFromTest(testInput test.Input) *ftwhttp.Request {
 	var req *ftwhttp.Request
 	// get raw request, if anything
-	raw, err := testRequest.GetRawRequest()
+	raw, err := testInput.GetRawRequest()
 	if err != nil {
 		log.Error().Msgf("ftw/run: error getting raw data: %s\n", err.Error())
 	}
 
 	// If we use raw or encoded request, then we don't use other fields
 	if raw != nil {
-		req = ftwhttp.NewRawRequest(raw, !testRequest.StopMagic)
+		req = ftwhttp.NewRawRequest(raw, !*testInput.AutocompleteHeaders)
 	} else {
 		rline := &ftwhttp.RequestLine{
-			Method:  testRequest.GetMethod(),
-			URI:     testRequest.GetURI(),
-			Version: testRequest.GetVersion(),
+			Method:  testInput.GetMethod(),
+			URI:     testInput.GetURI(),
+			Version: testInput.GetVersion(),
 		}
 
-		data := testRequest.ParseData()
+		data := testInput.ParseData()
 		// create a new request
-		req = ftwhttp.NewRequest(rline, testRequest.Headers,
-			data, !testRequest.StopMagic)
+		req = ftwhttp.NewRequest(rline, testInput.Headers,
+			data, !*testInput.AutocompleteHeaders)
 
 	}
 	return req
 }
 
-// applyInputOverride will check if config had global overrides and write that into the test.
-func applyInputOverride(o config.FTWTestOverride, testRequest *test.Input) error {
-	overrides := o.Overrides
-
-	if overrides.DestAddr != nil {
-		testRequest.DestAddr = overrides.DestAddr
-		if testRequest.Headers == nil {
-			testRequest.Headers = ftwhttp.Header{}
-		}
-		if overrides.OverrideEmptyHostHeader && testRequest.Headers.Get("Host") == "" {
-			testRequest.Headers.Set("Host", *overrides.DestAddr)
-		}
-	}
-
-	if overrides.Port != nil {
-		testRequest.Port = overrides.Port
-	}
-
-	if overrides.Protocol != nil {
-		testRequest.Protocol = overrides.Protocol
-	}
-
-	if overrides.URI != nil {
-		testRequest.URI = overrides.URI
-	}
-
-	if overrides.Version != nil {
-		testRequest.Version = overrides.Version
-	}
-
-	if overrides.Headers != nil {
-		if testRequest.Headers == nil {
-			testRequest.Headers = ftwhttp.Header{}
-		}
-		for k, v := range overrides.Headers {
-			testRequest.Headers.Set(k, v)
-		}
-	}
-
-	if overrides.Method != nil {
-		testRequest.Method = overrides.Method
-	}
-
-	if overrides.Data != nil {
-		testRequest.Data = overrides.Data
-	}
-
-	if overrides.StopMagic != nil {
-		testRequest.StopMagic = *overrides.StopMagic
-	}
-
-	if overrides.EncodedRequest != nil {
-		testRequest.EncodedRequest = *overrides.EncodedRequest
-	}
-
-	if overrides.RAWRequest != nil {
-		testRequest.RAWRequest = *overrides.RAWRequest
-	}
-
-	return nil
-}
-
 func notRunningInCloudMode(c *check.FTWCheck) bool {
 	return !c.CloudMode()
 }