diff --git a/README.md b/README.md index b16a50d2..505ac244 100644 --- a/README.md +++ b/README.md @@ -321,7 +321,7 @@ Tests can be altered using four lists: - `headers`: overrides headers, the format is a map of strings - `method`: overrides the method used to perform the request - `data`: overrides data sent in the request - - `stopmagic`: prevent header autocompletion (currently sets `Connection: close` and `Content-Length` for requests with body data) + - `autocomplete_headers`: prevent header autocompletion (currently sets `Connection: close` and `Content-Length` for requests with body data) - `encodedrequest`: overrides base64 encoded request - `rawrequest`: permits to provide a raw request. `method`, `uri` and `version` values will be ignored - `ignore` is for tests you want to ignore. You should add a comment on why you ignore the test diff --git a/check/base.go b/check/base.go index 548dacdd..b6244c11 100644 --- a/check/base.go +++ b/check/base.go @@ -47,7 +47,7 @@ func (c *FTWCheck) SetExpectResponse(response string) { // SetExpectError sets the boolean if we are expecting an error from the server func (c *FTWCheck) SetExpectError(expect bool) { - c.expected.ExpectError = expect + c.expected.ExpectError = &expect } // SetLogContains sets the string to look for in logs diff --git a/check/base_test.go b/check/base_test.go index 503cf3e8..451e7646 100644 --- a/check/base_test.go +++ b/check/base_test.go @@ -62,11 +62,11 @@ func (s *checkBaseTestSuite) TestNewCheck() { ResponseContains: "", LogContains: "nothing", NoLogContains: "", - ExpectError: true, + ExpectError: func() *bool { b := true; return &b }(), } c.SetExpectTestOutput(&to) - s.True(c.expected.ExpectError, "Problem setting expected output") + s.True(*c.expected.ExpectError, "Problem setting expected output") c.SetNoLogContains("nologcontains") diff --git a/check/error.go b/check/error.go index 04fce446..118d3f46 100644 --- a/check/error.go +++ b/check/error.go @@ -5,11 +5,11 @@ import "github.com/rs/zerolog/log" // AssertExpectError helper to check if this error was expected or not func (c *FTWCheck) AssertExpectError(err error) bool { if err != nil { - log.Debug().Msgf("ftw/check: expected error? -> %t, and error is %s", c.expected.ExpectError, err.Error()) + log.Debug().Msgf("ftw/check: expected error? -> %t, and error is %s", *c.expected.ExpectError, err.Error()) } else { - log.Debug().Msgf("ftw/check: expected error? -> %t, and error is nil", c.expected.ExpectError) + log.Debug().Msgf("ftw/check: expected error? -> %t, and error is nil", *c.expected.ExpectError) } - if c.expected.ExpectError && err != nil { + if *c.expected.ExpectError && err != nil { return true } return false diff --git a/cmd/root.go b/cmd/root.go index 4f832048..29caed95 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -68,4 +68,5 @@ func initConfig() { if cloud { cfg.RunMode = config.CloudRunMode } + } diff --git a/config/config.go b/config/config.go index 10f9e814..dd1f454c 100644 --- a/config/config.go +++ b/config/config.go @@ -35,17 +35,12 @@ func NewCloudConfig() *FTWConfiguration { // NewConfigFromFile reads configuration information from the config file if it exists, // or uses `.ftw.yaml` as default file func NewConfigFromFile(cfgFile string) (*FTWConfiguration, error) { - // Global koanf instance. Use "." as the key path delimiter. This can be "/" or any character. - var k = koanf.New(".") - var err error - cfg := NewDefaultConfig() - // first check if we had an explicit call with config file if cfgFile == "" { cfgFile = ".ftw.yaml" } - _, err = os.Stat(cfgFile) + _, err := os.Stat(cfgFile) if err != nil { // file does not exist, so we try the home folder var home string @@ -64,28 +59,19 @@ func NewConfigFromFile(cfgFile string) (*FTWConfiguration, error) { return nil, err } + k := getKoanfInstance() err = k.Load(file.Provider(cfgFile), yaml.Parser()) if err != nil { return nil, err } - // At this point we have loaded our config, now we need to - // unmarshal the whole root module - err = k.UnmarshalWithConf("", cfg, koanf.UnmarshalConf{Tag: "koanf"}) - if err != nil { - return nil, err - } - - return cfg, err + return unmarshal(k) } // NewConfigFromEnv reads configuration information from environment variables that start with `FTW_` func NewConfigFromEnv() (*FTWConfiguration, error) { - var err error - var k = koanf.New(".") - cfg := NewDefaultConfig() - - err = k.Load(env.Provider("FTW_", ".", func(s string) string { + k := getKoanfInstance() + err := k.Load(env.Provider("FTW_", ".", func(s string) string { return strings.ReplaceAll(strings.ToLower( strings.TrimPrefix(s, "FTW_")), "_", ".") }), nil) @@ -93,27 +79,19 @@ func NewConfigFromEnv() (*FTWConfiguration, error) { if err != nil { return nil, err } - // Unmarshal the whole root module - err = k.UnmarshalWithConf("", cfg, koanf.UnmarshalConf{Tag: "koanf"}) - return cfg, err + return unmarshal(k) } // NewConfigFromString initializes the configuration from a yaml formatted string. Useful for testing. func NewConfigFromString(conf string) (*FTWConfiguration, error) { - var k = koanf.New(".") - var err error - cfg := NewDefaultConfig() - - err = k.Load(rawbytes.Provider([]byte(conf)), yaml.Parser()) + k := getKoanfInstance() + err := k.Load(rawbytes.Provider([]byte(conf)), yaml.Parser()) if err != nil { return nil, err } - // Unmarshal the whole root module - err = k.UnmarshalWithConf("", cfg, koanf.UnmarshalConf{Tag: "koanf"}) - - return cfg, err + return unmarshal(k) } // WithLogfile changes the logfile in the configuration. @@ -145,3 +123,20 @@ func (c *FTWConfiguration) WithMaxMarkerRetries(retries int) { func (c *FTWConfiguration) WithMaxMarkerLogLines(amount int) { c.MaxMarkerLogLines = amount } + +// Unmarshal the loaded koanf instance into a configuration object +func unmarshal(k *koanf.Koanf) (*FTWConfiguration, error) { + config := NewDefaultConfig() + err := k.UnmarshalWithConf("", config, koanf.UnmarshalConf{Tag: "koanf"}) + if err != nil { + return nil, err + } + + return config, nil +} + +// Get the global koanf instance +func getKoanfInstance() *koanf.Koanf { + // Use "." as the key path delimiter. This can be "/" or any character. + return koanf.New(".") +} diff --git a/ftwhttp/types.go b/ftwhttp/types.go index 7682927b..31610bf3 100644 --- a/ftwhttp/types.go +++ b/ftwhttp/types.go @@ -62,7 +62,7 @@ type RequestLine struct { } // Request represents a request -// No Defaults represents the previous "stop_magic" behavior +// This struct without defaults represents the previous "autocomplete headers" behavior type Request struct { requestLine *RequestLine headers Header diff --git a/runner/run.go b/runner/run.go index 27a714f9..68914f0b 100644 --- a/runner/run.go +++ b/runner/run.go @@ -19,7 +19,7 @@ import ( "github.com/coreruleset/go-ftw/waflog" ) -var errBadTestRequest = errors.New("ftw/run: bad test: choose between data, encoded_request, or raw_request") +var errBadTestInput = errors.New("ftw/run: bad test input: choose between data, encoded_request, or raw_request") // Run runs your tests with the specified Config. func Run(cfg *config.FTWConfiguration, tests []test.FTWTest, c RunnerConfig, out *output.Output) (*TestRunContext, error) { @@ -75,9 +75,9 @@ func RunTest(runContext *TestRunContext, ftwTest test.FTWTest) error { for _, testCase := range ftwTest.Tests { // if we received a particular testid, skip until we find it - if needToSkipTest(runContext.Include, runContext.Exclude, testCase.TestTitle, ftwTest.Meta.Enabled) { + if needToSkipTest(runContext.Include, runContext.Exclude, testCase.TestTitle, *ftwTest.Meta.Enabled) { runContext.Stats.addResultToStats(Skipped, testCase.TestTitle, 0) - if !ftwTest.Meta.Enabled && !runContext.ShowOnlyFailed { + if !*ftwTest.Meta.Enabled && !runContext.ShowOnlyFailed { runContext.Output.Println("\tskipping %s - (enabled: false) in file.", testCase.TestTitle) } continue @@ -115,16 +115,13 @@ func RunStage(runContext *TestRunContext, ftwCheck *check.FTWCheck, testCase tes stageStartTime := time.Now() stageID := uuid.NewString() // Apply global overrides initially - testRequest := stage.Input - err := applyInputOverride(runContext.Config.TestOverride, &testRequest) - if err != nil { - log.Debug().Msgf("ftw/run: problem overriding input: %s", err.Error()) - } + testInput := stage.Input + test.ApplyInputOverrides(&runContext.Config.TestOverride.Overrides, &testInput) expectedOutput := stage.Output // Check sanity first - if checkTestSanity(testRequest) { - return errBadTestRequest + if checkTestSanity(testInput) { + return errBadTestInput } // Do not even run test if result is overridden. Just use the override and display the overridden result. @@ -138,24 +135,24 @@ func RunStage(runContext *TestRunContext, ftwCheck *check.FTWCheck, testCase tes // Destination is needed for a request dest := &ftwhttp.Destination{ - DestAddr: testRequest.GetDestAddr(), - Port: testRequest.GetPort(), - Protocol: testRequest.GetProtocol(), + DestAddr: testInput.GetDestAddr(), + Port: testInput.GetPort(), + Protocol: testInput.GetProtocol(), } if notRunningInCloudMode(ftwCheck) { startMarker, err := markAndFlush(runContext, dest, stageID) - if err != nil && !expectedOutput.ExpectError { + if err != nil && !*expectedOutput.ExpectError { return fmt.Errorf("failed to find start marker: %w", err) } ftwCheck.SetStartMarker(startMarker) } - req = getRequestFromTest(testRequest) + req = getRequestFromTest(testInput) - err = runContext.Client.NewConnection(*dest) + err := runContext.Client.NewConnection(*dest) - if err != nil && !expectedOutput.ExpectError { + if err != nil && !*expectedOutput.ExpectError { return fmt.Errorf("can't connect to destination %+v: %w", dest, err) } runContext.Client.StartTrackingTime() @@ -163,13 +160,13 @@ func RunStage(runContext *TestRunContext, ftwCheck *check.FTWCheck, testCase tes response, responseErr := runContext.Client.Do(*req) runContext.Client.StopTrackingTime() - if responseErr != nil && !expectedOutput.ExpectError { + if responseErr != nil && !*expectedOutput.ExpectError { return fmt.Errorf("failed sending request to destination %+v: %w", dest, responseErr) } if notRunningInCloudMode(ftwCheck) { endMarker, err := markAndFlush(runContext, dest, stageID) - if err != nil && !expectedOutput.ExpectError { + if err != nil && !*expectedOutput.ExpectError { return fmt.Errorf("failed to find end marker: %w", err) } @@ -270,10 +267,10 @@ func needToSkipTest(include *regexp.Regexp, exclude *regexp.Regexp, title string return result } -func checkTestSanity(testRequest test.Input) bool { - return (utils.IsNotEmpty(testRequest.Data) && testRequest.EncodedRequest != "") || - (utils.IsNotEmpty(testRequest.Data) && testRequest.RAWRequest != "") || - (testRequest.EncodedRequest != "" && testRequest.RAWRequest != "") +func checkTestSanity(testInput test.Input) bool { + return (utils.IsNotEmpty(testInput.Data) && testInput.EncodedRequest != "") || + (utils.IsNotEmpty(testInput.Data) && testInput.RAWRequest != "") || + (testInput.EncodedRequest != "" && testInput.RAWRequest != "") } func displayResult(rc *TestRunContext, result TestResult, roundTripTime time.Duration, stageTime time.Duration) { @@ -353,95 +350,33 @@ func checkResult(c *check.FTWCheck, response *ftwhttp.Response, responseError er return Success } -func getRequestFromTest(testRequest test.Input) *ftwhttp.Request { +func getRequestFromTest(testInput test.Input) *ftwhttp.Request { var req *ftwhttp.Request // get raw request, if anything - raw, err := testRequest.GetRawRequest() + raw, err := testInput.GetRawRequest() if err != nil { log.Error().Msgf("ftw/run: error getting raw data: %s\n", err.Error()) } // If we use raw or encoded request, then we don't use other fields if raw != nil { - req = ftwhttp.NewRawRequest(raw, !testRequest.StopMagic) + req = ftwhttp.NewRawRequest(raw, !*testInput.AutocompleteHeaders) } else { rline := &ftwhttp.RequestLine{ - Method: testRequest.GetMethod(), - URI: testRequest.GetURI(), - Version: testRequest.GetVersion(), + Method: testInput.GetMethod(), + URI: testInput.GetURI(), + Version: testInput.GetVersion(), } - data := testRequest.ParseData() + data := testInput.ParseData() // create a new request - req = ftwhttp.NewRequest(rline, testRequest.Headers, - data, !testRequest.StopMagic) + req = ftwhttp.NewRequest(rline, testInput.Headers, + data, !*testInput.AutocompleteHeaders) } return req } -// applyInputOverride will check if config had global overrides and write that into the test. -func applyInputOverride(o config.FTWTestOverride, testRequest *test.Input) error { - overrides := o.Overrides - - if overrides.DestAddr != nil { - testRequest.DestAddr = overrides.DestAddr - if testRequest.Headers == nil { - testRequest.Headers = ftwhttp.Header{} - } - if overrides.OverrideEmptyHostHeader && testRequest.Headers.Get("Host") == "" { - testRequest.Headers.Set("Host", *overrides.DestAddr) - } - } - - if overrides.Port != nil { - testRequest.Port = overrides.Port - } - - if overrides.Protocol != nil { - testRequest.Protocol = overrides.Protocol - } - - if overrides.URI != nil { - testRequest.URI = overrides.URI - } - - if overrides.Version != nil { - testRequest.Version = overrides.Version - } - - if overrides.Headers != nil { - if testRequest.Headers == nil { - testRequest.Headers = ftwhttp.Header{} - } - for k, v := range overrides.Headers { - testRequest.Headers.Set(k, v) - } - } - - if overrides.Method != nil { - testRequest.Method = overrides.Method - } - - if overrides.Data != nil { - testRequest.Data = overrides.Data - } - - if overrides.StopMagic != nil { - testRequest.StopMagic = *overrides.StopMagic - } - - if overrides.EncodedRequest != nil { - testRequest.EncodedRequest = *overrides.EncodedRequest - } - - if overrides.RAWRequest != nil { - testRequest.RAWRequest = *overrides.RAWRequest - } - - return nil -} - func notRunningInCloudMode(c *check.FTWCheck) bool { return !c.CloudMode() } diff --git a/runner/run_input_override_test.go b/runner/run_input_override_test.go index d04f4f60..9a41dea3 100644 --- a/runner/run_input_override_test.go +++ b/runner/run_input_override_test.go @@ -3,6 +3,7 @@ package runner import ( "bytes" "errors" + "fmt" "runtime" "strconv" "strings" @@ -27,6 +28,7 @@ var configTemplate = ` testoverride: input: {{ with .StopMagic }}stop_magic: {{ . }}{{ end }} + {{ with .AutocompleteHeaders }}autocomplete_headers: {{ . }}{{ end }} {{ with .BrokenConfig }}this_does_not_exist: "test"{{ end }} {{ with .Port }}port: {{ . }}{{ end }} {{ with .DestAddr }}dest_addr: {{ . }}{{ end }} @@ -93,6 +95,9 @@ var overrideConfigMap = map[string]interface{}{ "TestApplyInputOverrideStopMagic": map[string]interface{}{ "StopMagic": "true", }, + "TestApplyInputOverrideAutocompleteHeaders": map[string]interface{}{ + "AutocompleteHeaders": "true", + }, } // getOverrideConfigValue is useful to not repeat the text in the test itself @@ -109,7 +114,12 @@ func getOverrideConfigValue(key string) (string, error) { keyParts := strings.Split(key, ".") return overrideConfigMap[name].(map[string]interface{})[keyParts[0]].(map[string]string)[keyParts[1]], nil } - return overrideConfigMap[name].(map[string]interface{})[key].(string), nil + value, ok := overrideConfigMap[name].(map[string]interface{})[key] + if !ok { + return "", fmt.Errorf("Key '%s' not found four test '%s'", key, name) + } + + return value.(string), nil } return "", errors.New("failed to determine calling function") } @@ -150,13 +160,12 @@ func (s *inputOverrideTestSuite) TestSetHostFromDestAddr() { TestOverride: config.FTWTestOverride{ Overrides: test.Overrides{ DestAddr: &overrideHost, - OverrideEmptyHostHeader: true, + OverrideEmptyHostHeader: func() *bool { b := true; return &b }(), }, }, } - err = applyInputOverride(cfg.TestOverride, &testInput) - s.Require().NoError(err, "Failed to apply input overrides") + test.ApplyInputOverrides(&cfg.TestOverride.Overrides, &testInput) s.Equal(overrideHost, *testInput.DestAddr, "`dest_addr` should have been overridden") @@ -176,8 +185,7 @@ func (s *inputOverrideTestSuite) TestSetHostFromHostHeaderOverride() { DestAddr: &originalDestAddr, } - err = applyInputOverride(s.cfg.TestOverride, &testInput) - s.Require().NoError(err, "Failed to apply input overrides") + test.ApplyInputOverrides(&s.cfg.TestOverride.Overrides, &testInput) hostHeader := testInput.Headers.Get("Host") s.NotEqual("", hostHeader, "Host header must be set after overriding the `Host` header") @@ -199,8 +207,7 @@ func (s *inputOverrideTestSuite) TestSetHeaderOverridingExistingOne() { s.NotNil(testInput.Headers, "Header map must exist before overriding any header") - err = applyInputOverride(s.cfg.TestOverride, &testInput) - s.Require().NoError(err, "Failed to apply input overrides") + test.ApplyInputOverrides(&s.cfg.TestOverride.Overrides, &testInput) overriddenHeader := testInput.Headers.Get("unique_id") s.NotEqual("", overriddenHeader, "unique_id header must be set after overriding it") @@ -218,8 +225,7 @@ func (s *inputOverrideTestSuite) TestApplyInputOverrides() { s.NotNil(testInput.Headers, "Header map must exist before overriding any header") - err = applyInputOverride(s.cfg.TestOverride, &testInput) - s.Require().NoError(err, "Failed to apply input overrides") + test.ApplyInputOverrides(&s.cfg.TestOverride.Overrides, &testInput) overriddenHeader := testInput.Headers.Get("unique_id") s.NotEqual("", overriddenHeader, "unique_id header must be set after overriding it") @@ -235,8 +241,8 @@ func (s *inputOverrideTestSuite) TestApplyInputOverrideURI() { URI: &originalURI, } - err = applyInputOverride(s.cfg.TestOverride, &testInput) - s.Require().NoError(err, "Failed to apply input overrides") + test.ApplyInputOverrides(&s.cfg.TestOverride.Overrides, &testInput) + s.Equal(overrideURI, *testInput.URI, "`URI` should have been overridden") } @@ -248,8 +254,8 @@ func (s *inputOverrideTestSuite) TestApplyInputOverrideVersion() { testInput := test.Input{ Version: &originalVersion, } - err = applyInputOverride(s.cfg.TestOverride, &testInput) - s.Require().NoError(err, "Failed to apply input overrides") + test.ApplyInputOverrides(&s.cfg.TestOverride.Overrides, &testInput) + s.Equal(overrideVersion, *testInput.Version, "`Version` should have been overridden") } @@ -261,8 +267,8 @@ func (s *inputOverrideTestSuite) TestApplyInputOverrideMethod() { testInput := test.Input{ Method: &originalMethod, } - err = applyInputOverride(s.cfg.TestOverride, &testInput) - s.Require().NoError(err, "Failed to apply input overrides") + test.ApplyInputOverrides(&s.cfg.TestOverride.Overrides, &testInput) + s.Equal(overrideMethod, *testInput.Method, "`Method` should have been overridden") } @@ -274,8 +280,8 @@ func (s *inputOverrideTestSuite) TestApplyInputOverrideData() { testInput := test.Input{ Data: &originalData, } - err = applyInputOverride(s.cfg.TestOverride, &testInput) - s.Require().NoError(err, "Failed to apply input overrides") + test.ApplyInputOverrides(&s.cfg.TestOverride.Overrides, &testInput) + s.Equal(overrideData, *testInput.Data, "`Data` should have been overridden") } @@ -285,11 +291,26 @@ func (s *inputOverrideTestSuite) TestApplyInputOverrideStopMagic() { overrideStopMagic, err := strconv.ParseBool(stopMagicBool) s.Require().NoError(err, "Failed to parse `StopMagic` override value") testInput := test.Input{ - StopMagic: false, + StopMagic: func() *bool { b := false; return &b }(), } - err = applyInputOverride(s.cfg.TestOverride, &testInput) - s.Require().NoError(err, "Failed to apply input overrides") - s.Equal(overrideStopMagic, testInput.StopMagic, "`StopMagic` should have been overridden") + test.ApplyInputOverrides(&s.cfg.TestOverride.Overrides, &testInput) + + // nolint + s.Equal(overrideStopMagic, *testInput.StopMagic, "`StopMagic` should have been overridden") +} + +func (s *inputOverrideTestSuite) TestApplyInputOverrideAutocompleteHeaders() { + autocompleteHeadersBool, err := getOverrideConfigValue("AutocompleteHeaders") + s.NoError(err, "cannot get override value") + overrideAutocompleteHeaders, err := strconv.ParseBool(autocompleteHeadersBool) + s.NoError(err, "Failed to parse `AutocompleteHeaders` override value") + testInput := test.Input{ + AutocompleteHeaders: func() *bool { b := false; return &b }(), + } + test.ApplyInputOverrides(&s.cfg.TestOverride.Overrides, &testInput) + + // nolint + s.Equal(overrideAutocompleteHeaders, *testInput.AutocompleteHeaders, "`AutocompleteHeaders` should have been overridden") } func (s *inputOverrideTestSuite) TestApplyInputOverrideEncodedRequest() { @@ -299,8 +320,8 @@ func (s *inputOverrideTestSuite) TestApplyInputOverrideEncodedRequest() { testInput := test.Input{ EncodedRequest: originalEncodedRequest, } - err = applyInputOverride(s.cfg.TestOverride, &testInput) - s.Require().NoError(err, "Failed to apply input overrides") + test.ApplyInputOverrides(&s.cfg.TestOverride.Overrides, &testInput) + s.NoError(err, "Failed to apply input overrides") s.Equal(overrideEncodedRequest, testInput.EncodedRequest, "`EncodedRequest` should have been overridden") } @@ -313,7 +334,7 @@ func (s *inputOverrideTestSuite) TestApplyInputOverrideRAWRequest() { RAWRequest: originalRAWRequest, } - err = applyInputOverride(s.cfg.TestOverride, &testInput) - s.Require().NoError(err, "Failed to apply input overrides") + test.ApplyInputOverrides(&s.cfg.TestOverride.Overrides, &testInput) + s.Equal(overrideRAWRequest, testInput.RAWRequest, "`RAWRequest` should have been overridden") } diff --git a/test/data_test.go b/test/data_test.go index e8321417..50369c63 100644 --- a/test/data_test.go +++ b/test/data_test.go @@ -29,13 +29,13 @@ Host: "localhost" Content-Type: "application/x-www-form-urlencoded" data: "hi=test" protocol: "http" -stop_magic: true +autocomplete_headers: true uri: "/" ` input := Input{} err := yaml.Unmarshal([]byte(yamlString), &input) s.Require().NoError(err) - s.True(input.StopMagic) + s.True(*input.AutocompleteHeaders) } func (s *dataTestSuite) TestGetPartialDataFromYAML() { @@ -50,13 +50,14 @@ Content-Type: "application/x-www-form-urlencoded" data: "hi=test" version: "" protocol: "http" -stop_magic: true +autocomplete_headers: false uri: "/" ` input := Input{} err := yaml.Unmarshal([]byte(yamlString), &input) s.Require().NoError(err) s.Empty(*input.Version) + s.False(*input.AutocompleteHeaders) } func (s *dataTestSuite) TestDataTemplateFromYAML() { @@ -71,7 +72,7 @@ Content-Type: "application/x-www-form-urlencoded" data: 'foo=%3d{{ "+" | repeat 34 }}' version: "" protocol: "http" -stop_magic: true +autocomplete_headers: true uri: "/" ` input := Input{} @@ -81,4 +82,6 @@ uri: "/" s.Require().NoError(err) data = input.ParseData() s.Equal([]byte(repeatTestSprig), data) + + s.True(*input.AutocompleteHeaders) } diff --git a/test/defaults_test.go b/test/defaults_test.go index 074da0d0..71c3c401 100644 --- a/test/defaults_test.go +++ b/test/defaults_test.go @@ -21,10 +21,10 @@ func getTestInputDefaults() *Input { data := "My Data" inputDefaults := Input{ - Headers: make(ftwhttp.Header), - Data: &data, - SaveCookie: false, - StopMagic: false, + Headers: make(ftwhttp.Header), + Data: &data, + SaveCookie: func() *bool { b := false; return &b }(), + AutocompleteHeaders: func() *bool { b := false; return &b }(), } return &inputDefaults } @@ -38,17 +38,17 @@ func getTestExampleInput() *Input { version := "HTTP/1.1" inputTest := Input{ - DestAddr: &destaddr, - Port: &port, - Protocol: &protocol, - URI: &uri, - Version: &version, - Headers: make(ftwhttp.Header), - Method: &method, - Data: nil, - EncodedRequest: "TXkgRGF0YQo=", - SaveCookie: false, - StopMagic: false, + DestAddr: &destaddr, + Port: &port, + Protocol: &protocol, + URI: &uri, + Version: &version, + Headers: make(ftwhttp.Header), + Method: &method, + Data: nil, + EncodedRequest: "TXkgRGF0YQo=", + SaveCookie: func() *bool { b := false; return &b }(), + AutocompleteHeaders: func() *bool { b := false; return &b }(), } return &inputTest @@ -75,8 +75,8 @@ Keep-Alive: 300 Proxy-Connection: keep-alive User-Agent: Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; SV1; .NET CLR 2.0.50727) `, - SaveCookie: false, - StopMagic: true, + SaveCookie: func() *bool { b := false; return &b }(), + AutocompleteHeaders: func() *bool { b := true; return &b }(), } return &inputTest @@ -128,7 +128,7 @@ func (s *defaultsTestSuite) TestDefaultGetters() { func (s *defaultsTestSuite) TestRaw() { raw := getRawInput() - s.True(raw.StopMagic) + s.True(*raw.AutocompleteHeaders) request, _ := raw.GetRawRequest() s.NotEqual(2, bytes.Index(request, []byte("Acunetix"))) diff --git a/test/files.go b/test/files.go index 29f76d75..40321a41 100644 --- a/test/files.go +++ b/test/files.go @@ -3,7 +3,6 @@ package test import ( "errors" "os" - "regexp" "github.com/goccy/go-yaml" "github.com/rs/zerolog/log" @@ -35,12 +34,12 @@ func GetTestsFromFiles(globPattern string) ([]FTWTest, error) { if err != nil { log.Error().Msgf("Problem detected in file %s:\n%s\n%s", fileName, yaml.FormatError(err, true, true), - describeYamlError(err)) + DescribeYamlError(err)) return tests, err } ftwTest.FileName = fileName - tests = append(tests, ftwTest) + tests = append(tests, *ftwTest) } if len(tests) == 0 { @@ -49,21 +48,6 @@ func GetTestsFromFiles(globPattern string) ([]FTWTest, error) { return tests, nil } -// GetTestFromYaml will get the tests to be processed from a YAML string. -func GetTestFromYaml(testYaml []byte) (ftwTest FTWTest, err error) { - ftwTest, err = readTestYaml(testYaml) - if err != nil { - return FTWTest{}, err - } - - return ftwTest, nil -} - -func readTestYaml(testYaml []byte) (t FTWTest, err error) { - err = yaml.Unmarshal([]byte(testYaml), &t) - return t, err -} - func readFileContents(fileName string) (contents []byte, err error) { contents, err = os.ReadFile(fileName) if err != nil { @@ -71,37 +55,3 @@ func readFileContents(fileName string) (contents []byte, err error) { } return contents, err } - -func describeYamlError(yamlError error) string { - matched, err := regexp.MatchString(`.*int was used where sequence is expected.*`, yamlError.Error()) - if err != nil { - return err.Error() - } - if matched { - return "\nTip: This might refer to a \"status\" line being '200', where it should be '[200]'.\n" + - "The default \"status\" is a list now.\n" + - "A simple example would be like this:\n\n" + - "status: 403\n" + - "needs to be changed to:\n\n" + - "status: [403]\n\n" - } - matched, err = regexp.MatchString(`.*cannot unmarshal \[]interface {} into Go struct field FTWTest.Tests of type string.*`, yamlError.Error()) - if err != nil { - return err.Error() - } - if matched { - return "\nTip: This might refer to \"data\" on the test being a list of strings instead of a proper YAML multiline.\n" + - "To fix this, convert this \"data\" string list to a multiline YAML and this will be fixed.\n" + - "A simple example would be like this:\n\n" + - "data:\n" + - " - 'Hello'\n" + - " - 'World'\n" + - "can be expressed as:\n\n" + - "data: |\n" + - " Hello\n" + - " World\n\n" + - "You can also remove single/double quotes from beggining and end of text, they are not needed. See https://yaml-multiline.info/ for additional help.\n" - } - - return "We do not have an extended explanation of this error." -} diff --git a/test/files_test.go b/test/files_test.go index f8e26f39..3219dd1c 100644 --- a/test/files_test.go +++ b/test/files_test.go @@ -23,6 +23,7 @@ var yamlTest = ` - stage: input: + autocomplete_headers: false dest_addr: "127.0.0.1" port: 80 headers: diff --git a/test/types.go b/test/types.go index d0ec9525..e37036a6 100644 --- a/test/types.go +++ b/test/types.go @@ -1,40 +1,46 @@ package test -import "github.com/coreruleset/go-ftw/ftwhttp" +import ( + "github.com/coreruleset/go-ftw/ftwhttp" +) // Input represents the input request in a stage -// The fields `Version`, `Method` and `URI` we want to explicitly now when they are set to "" +// The fields `Version`, `Method` and `URI` we want to explicitly know when they are set to "" type Input struct { - DestAddr *string `yaml:"dest_addr,omitempty" koanf:"dest_addr,omitempty"` - Port *int `yaml:"port,omitempty" koanf:"port,omitempty"` - Protocol *string `yaml:"protocol,omitempty" koanf:"protocol,omitempty"` - URI *string `yaml:"uri,omitempty" koanf:"uri,omitempty"` - Version *string `yaml:"version,omitempty" koanf:"version,omitempty"` - Headers ftwhttp.Header `yaml:"headers,omitempty" koanf:"headers,omitempty"` - Method *string `yaml:"method,omitempty" koanf:"method,omitempty"` - Data *string `yaml:"data,omitempty" koanf:"data,omitempty"` - SaveCookie bool `yaml:"save_cookie,omitempty" koanf:"save_cookie,omitempty"` - StopMagic bool `yaml:"stop_magic" koanf:"stop_magic,omitempty"` - EncodedRequest string `yaml:"encoded_request,omitempty" koanf:"encoded_request,omitempty"` - RAWRequest string `yaml:"raw_request,omitempty" koanf:"raw_request,omitempty"` + DestAddr *string `yaml:"dest_addr,omitempty"` + Port *int `yaml:"port,omitempty"` + Protocol *string `yaml:"protocol,omitempty"` + URI *string `yaml:"uri,omitempty"` + Version *string `yaml:"version,omitempty"` + Headers ftwhttp.Header `yaml:"headers,omitempty"` + Method *string `yaml:"method,omitempty"` + Data *string `yaml:"data,omitempty"` + SaveCookie *bool `yaml:"save_cookie,omitempty"` + // Deprecated: replaced with AutocompleteHeaders + StopMagic *bool `yaml:"stop_magic"` + AutocompleteHeaders *bool `yaml:"autocomplete_headers"` + EncodedRequest string `yaml:"encoded_request,omitempty"` + RAWRequest string `yaml:"raw_request,omitempty"` } // Overrides represents the overridden inputs that have to be applied to tests type Overrides struct { - DestAddr *string `yaml:"dest_addr,omitempty" koanf:"dest_addr,omitempty"` - Port *int `yaml:"port,omitempty" koanf:"port,omitempty"` - Protocol *string `yaml:"protocol,omitempty" koanf:"protocol,omitempty"` - URI *string `yaml:"uri,omitempty" koanf:"uri,omitempty"` - Version *string `yaml:"version,omitempty" koanf:"version,omitempty"` - Headers ftwhttp.Header `yaml:"headers,omitempty" koanf:"headers,omitempty"` - Method *string `yaml:"method,omitempty" koanf:"method,omitempty"` - Data *string `yaml:"data,omitempty" koanf:"data,omitempty"` - //SaveCookie bool `yaml:"save_cookie,omitempty" koanf:"save_cookie,omitempty"` + DestAddr *string `yaml:"dest_addr,omitempty" koanf:"dest_addr,omitempty"` + Port *int `yaml:"port,omitempty" koanf:"port,omitempty"` + Protocol *string `yaml:"protocol,omitempty" koanf:"protocol,omitempty"` + URI *string `yaml:"uri,omitempty" koanf:"uri,omitempty"` + Version *string `yaml:"version,omitempty" koanf:"version,omitempty"` + Headers ftwhttp.Header `yaml:"headers,omitempty" koanf:"headers,omitempty"` + Method *string `yaml:"method,omitempty" koanf:"method,omitempty"` + Data *string `yaml:"data,omitempty" koanf:"data,omitempty"` + SaveCookie *bool `yaml:"save_cookie,omitempty" koanf:"save_cookie,omitempty"` + // Deprecated: replaced with AutocompleteHeaders StopMagic *bool `yaml:"stop_magic" koanf:"stop_magic,omitempty"` + AutocompleteHeaders *bool `yaml:"autocomplete_headers" koanf:"autocomplete_headers,omitempty"` EncodedRequest *string `yaml:"encoded_request,omitempty" koanf:"encoded_request,omitempty"` RAWRequest *string `yaml:"raw_request,omitempty" koanf:"raw_request,omitempty"` - OverrideEmptyHostHeader bool `yaml:"override_empty_host_header,omitempty" koanf:"override_empty_host_header,omitempty"` + OverrideEmptyHostHeader *bool `yaml:"override_empty_host_header,omitempty" koanf:"override_empty_host_header,omitempty"` } // Output is the response expected from the test @@ -43,7 +49,7 @@ type Output struct { ResponseContains string `yaml:"response_contains,omitempty"` LogContains string `yaml:"log_contains,omitempty"` NoLogContains string `yaml:"no_log_contains,omitempty"` - ExpectError bool `yaml:"expect_error,omitempty"` + ExpectError *bool `yaml:"expect_error,omitempty"` } // Stage is an individual test stage @@ -66,9 +72,115 @@ type FTWTest struct { FileName string Meta struct { Author string `yaml:"author,omitempty"` - Enabled bool `yaml:"enabled,omitempty"` + Enabled *bool `yaml:"enabled,omitempty"` Name string `yaml:"name,omitempty"` Description string `yaml:"description,omitempty"` } `yaml:"meta"` Tests []Test `yaml:"tests"` } + +// ApplyInputOverride will check if config had global overrides and write that into the test. +func ApplyInputOverrides(overrides *Overrides, input *Input) { + applySimpleOverrides(overrides, input) + applyDestAddrOverride(overrides, input) + applyHeadersOverride(overrides, input) + postProcessAutocompleteHeaders(overrides.AutocompleteHeaders, overrides.StopMagic, input) +} + +func applyDestAddrOverride(overrides *Overrides, input *Input) { + if overrides.DestAddr != nil { + input.DestAddr = overrides.DestAddr + if input.Headers == nil { + input.Headers = ftwhttp.Header{} + } + if overrides.OverrideEmptyHostHeader != nil && *overrides.OverrideEmptyHostHeader && input.Headers.Get("Host") == "" { + input.Headers.Set("Host", *overrides.DestAddr) + } + } +} + +func applySimpleOverrides(overrides *Overrides, input *Input) { + if overrides.Port != nil { + input.Port = overrides.Port + } + + if overrides.Protocol != nil { + input.Protocol = overrides.Protocol + } + + if overrides.URI != nil { + input.URI = overrides.URI + } + + if overrides.Version != nil { + input.Version = overrides.Version + } + + if overrides.Method != nil { + input.Method = overrides.Method + } + + if overrides.Data != nil { + input.Data = overrides.Data + } + + if overrides.SaveCookie != nil { + input.SaveCookie = overrides.SaveCookie + } + + if overrides.EncodedRequest != nil { + input.EncodedRequest = *overrides.EncodedRequest + } + + if overrides.RAWRequest != nil { + input.RAWRequest = *overrides.RAWRequest + } +} + +func applyHeadersOverride(overrides *Overrides, input *Input) { + if overrides.Headers != nil { + if input.Headers == nil { + input.Headers = ftwhttp.Header{} + } + for k, v := range overrides.Headers { + input.Headers.Set(k, v) + } + } +} + +func postLoadTestFTWTest(ftwTest *FTWTest) { + for _, test := range ftwTest.Tests { + postLoadTest(&test) + } +} + +func postLoadTest(test *Test) { + for index := range test.Stages { + postLoadStage(&test.Stages[index].Stage) + } +} + +func postLoadStage(stage *Stage) { + postLoadInput(&stage.Input) +} + +func postLoadInput(input *Input) { + postProcessAutocompleteHeaders(input.AutocompleteHeaders, input.StopMagic, input) +} + +func postProcessAutocompleteHeaders(autocompleteHeaders *bool, stopMagic *bool, input *Input) { + autocompleteHeadersMissing := autocompleteHeaders == nil + stopMagicMissing := stopMagic == nil + // default value + finalValue := true + + if autocompleteHeadersMissing && !stopMagicMissing { + // StopMagic has the inverse boolean logic + finalValue = !*stopMagic + } else if !autocompleteHeadersMissing { + finalValue = *autocompleteHeaders + } + input.AutocompleteHeaders = &finalValue + // StopMagic has the inverse boolean logic + input.StopMagic = func() *bool { b := !finalValue; return &b }() +} diff --git a/test/types_test.go b/test/types_test.go new file mode 100644 index 00000000..ef1d6ae1 --- /dev/null +++ b/test/types_test.go @@ -0,0 +1,251 @@ +package test + +import ( + "testing" + + "github.com/stretchr/testify/suite" +) + +type typesTestSuite struct { + suite.Suite +} + +func TestTypesTestSuite(t *testing.T) { + suite.Run(t, new(typesTestSuite)) +} + +var autocompleteHeadersDefaultYaml = `--- +meta: + author: "tester" + enabled: true + name: "gotest-ftw.yaml" + description: "Example Test" +tests: + - test_title: "001" + description: "autocomplete headers by default" + stages: + - stage: + input: + dest_addr: "localhost" + headers: + User-Agent: "ModSecurity CRS 3 Tests" + Accept: "*/*" + Host: "localhost" + output: + expect_error: False + status: [200] + - test_title: "002" + description: "autocomplete headers by default" + stages: + - stage: + input: + stop_magic: true + dest_addr: "localhost" + headers: + User-Agent: "ModSecurity CRS 3 Tests" + Accept: "*/*" + Host: "localhost" + output: + expect_error: False + status: [200] + - test_title: "003" + description: "autocomplete headers by default" + stages: + - stage: + input: + stop_magic: false + dest_addr: "localhost" + headers: + User-Agent: "ModSecurity CRS 3 Tests" + Accept: "*/*" + Host: "localhost" + output: + expect_error: False + status: [200] +` + +var autocompleteHeadersFalseYaml = `--- +meta: + author: "tester" + enabled: true + name: "gotest-ftw.yaml" + description: "Example Test" +tests: + - test_title: "001" + description: "autocomplete headers explicitly" + stages: + - stage: + input: + autocomplete_headers: false + dest_addr: "localhost" + headers: + User-Agent: "ModSecurity CRS 3 Tests" + Accept: "*/*" + Host: "localhost" + output: + expect_error: False + status: [200] + - test_title: "002" + description: "autocomplete headers explicitly" + stages: + - stage: + input: + autocomplete_headers: false + stop_magic: true + dest_addr: "localhost" + headers: + User-Agent: "ModSecurity CRS 3 Tests" + Accept: "*/*" + Host: "localhost" + output: + expect_error: False + status: [200] + - test_title: "003" + description: "autocomplete headers explicitly" + stages: + - stage: + input: + autocomplete_headers: false + stop_magic: false + dest_addr: "localhost" + headers: + User-Agent: "ModSecurity CRS 3 Tests" + Accept: "*/*" + Host: "localhost" + output: + expect_error: False + status: [200] +` + +var autocompleteHeadersTrueYaml = `--- +meta: + author: "tester" + enabled: true + name: "gotest-ftw.yaml" + description: "Example Test" +tests: + - test_title: "001" + description: "do not autocomplete headers explicitly" + stages: + - stage: + input: + autocomplete_headers: true + dest_addr: "localhost" + headers: + User-Agent: "ModSecurity CRS 3 Tests" + Accept: "*/*" + Host: "localhost" + output: + expect_error: False + status: [200] + - test_title: "002" + description: "do not autocomplete headers explicitly" + stages: + - stage: + input: + autocomplete_headers: true + stop_magic: true + dest_addr: "localhost" + headers: + User-Agent: "ModSecurity CRS 3 Tests" + Accept: "*/*" + Host: "localhost" + output: + expect_error: False + status: [200] + - test_title: "003" + description: "do not autocomplete headers explicitly" + stages: + - stage: + input: + autocomplete_headers: true + stop_magic: false + dest_addr: "localhost" + headers: + User-Agent: "ModSecurity CRS 3 Tests" + Accept: "*/*" + Host: "localhost" + output: + expect_error: False + status: [200] +` + +func (s *typesTestSuite) TestAutocompleteHeadersDefault_StopMagicDefault() { + test, err := GetTestFromYaml([]byte(autocompleteHeadersDefaultYaml)) + s.NoError(err, "Parsing YAML shouldn't fail") + + input := test.Tests[0].Stages[0].Stage.Input + s.True(*input.AutocompleteHeaders) + s.False(*input.StopMagic) +} + +func (s *typesTestSuite) TestAutocompleteHeadersDefault_StopMagicTrue() { + test, err := GetTestFromYaml([]byte(autocompleteHeadersDefaultYaml)) + s.NoError(err, "Parsing YAML shouldn't fail") + + input := test.Tests[1].Stages[0].Stage.Input + s.False(*input.AutocompleteHeaders) + s.True(*input.StopMagic) +} +func (s *typesTestSuite) TestAutocompleteHeadersDefault_StopMagicFalse() { + test, err := GetTestFromYaml([]byte(autocompleteHeadersDefaultYaml)) + s.NoError(err, "Parsing YAML shouldn't fail") + + input := test.Tests[2].Stages[0].Stage.Input + s.True(*input.AutocompleteHeaders) + s.False(*input.StopMagic) +} + +func (s *typesTestSuite) TestAutocompleteHeadersFalse_StopMagicDefault() { + test, err := GetTestFromYaml([]byte(autocompleteHeadersFalseYaml)) + s.NoError(err, "Parsing YAML shouldn't fail") + + input := test.Tests[0].Stages[0].Stage.Input + s.False(*input.AutocompleteHeaders) + s.True(*input.StopMagic) +} + +func (s *typesTestSuite) TestAutocompleteHeadersFalse_StopMagicTrue() { + test, err := GetTestFromYaml([]byte(autocompleteHeadersFalseYaml)) + s.NoError(err, "Parsing YAML shouldn't fail") + + input := test.Tests[1].Stages[0].Stage.Input + s.False(*input.AutocompleteHeaders) + s.True(*input.StopMagic) +} + +func (s *typesTestSuite) TestAutocompleteHeadersFalse_StopMagicFalse() { + test, err := GetTestFromYaml([]byte(autocompleteHeadersFalseYaml)) + s.NoError(err, "Parsing YAML shouldn't fail") + + input := test.Tests[2].Stages[0].Stage.Input + s.False(*input.AutocompleteHeaders) + s.True(*input.StopMagic) +} + +func (s *typesTestSuite) TestAutocompleteHeadersTrue_StopMagicDefault() { + test, err := GetTestFromYaml([]byte(autocompleteHeadersTrueYaml)) + s.NoError(err, "Parsing YAML shouldn't fail") + + input := test.Tests[0].Stages[0].Stage.Input + s.True(*input.AutocompleteHeaders) + s.False(*input.StopMagic) +} + +func (s *typesTestSuite) TestAutocompleteHeadersTrue_StopMagicTrue() { + test, err := GetTestFromYaml([]byte(autocompleteHeadersTrueYaml)) + s.NoError(err, "Parsing YAML shouldn't fail") + + input := test.Tests[1].Stages[0].Stage.Input + s.True(*input.AutocompleteHeaders) + s.False(*input.StopMagic) +} + +func (s *typesTestSuite) TestAutocompleteHeadersTrue_StopMagicFalse() { + test, err := GetTestFromYaml([]byte(autocompleteHeadersTrueYaml)) + s.NoError(err, "Parsing YAML shouldn't fail") + + input := test.Tests[2].Stages[0].Stage.Input + s.True(*input.AutocompleteHeaders) + s.False(*input.StopMagic) +} diff --git a/test/yaml.go b/test/yaml.go new file mode 100644 index 00000000..693596b4 --- /dev/null +++ b/test/yaml.go @@ -0,0 +1,54 @@ +package test + +import ( + "regexp" + + "github.com/goccy/go-yaml" +) + +// GetTestFromYaml will get the tests to be processed from a YAML string. +func GetTestFromYaml(testYaml []byte) (ftwTest *FTWTest, err error) { + ftwTest = &FTWTest{} + err = yaml.Unmarshal(testYaml, ftwTest) + if err != nil { + return &FTWTest{}, err + } + + postLoadTestFTWTest(ftwTest) + + return ftwTest, nil +} + +func DescribeYamlError(yamlError error) string { + matched, err := regexp.MatchString(`.*int was used where sequence is expected.*`, yamlError.Error()) + if err != nil { + return err.Error() + } + if matched { + return "\nTip: This might refer to a \"status\" line being '200', where it should be '[200]'.\n" + + "The default \"status\" is a list now.\n" + + "A simple example would be like this:\n\n" + + "status: 403\n" + + "needs to be changed to:\n\n" + + "status: [403]\n\n" + } + matched, err = regexp.MatchString(`.*cannot unmarshal \[]interface {} into Go struct field FTWTest.Tests of type string.*`, yamlError.Error()) + if err != nil { + return err.Error() + } + if matched { + return "\nTip: This might refer to \"data\" on the test being a list of strings instead of a proper YAML multiline.\n" + + "To fix this, convert this \"data\" string list to a multiline YAML and this will be fixed.\n" + + "A simple example would be like this:\n\n" + + "data:\n" + + " - 'Hello'\n" + + " - 'World'\n" + + "can be expressed as:\n\n" + + "data: |\n" + + " Hello\n" + + " World\n\n" + + "You can also remove single/double quotes from beggining and end of text, they are not needed. See https://yaml-multiline.info/ for additional help.\n" + } + + return "We do not have an extended explanation of this error." +}