Merge branch 'main' into use-koanf-v1

go.mod

@@ -1,9 +1,10 @@
 module github.com/coreruleset/go-ftw
 
-go 1.18
+go 1.21
 
 require (
 	github.com/Masterminds/sprig v2.22.0+incompatible
+	github.com/coreruleset/ftw-tests-schema v1.1.0
 	github.com/go-logr/zerologr v1.2.3
 	github.com/goccy/go-yaml v1.9.2
 	github.com/google/uuid v1.4.0

go.sum

@@ -43,6 +43,8 @@ github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnht
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
+github.com/coreruleset/ftw-tests-schema v1.1.0 h1:3+NYrdLE3HVmOc3nGrisRBBvY9lGjePUrV+YkT5Ay3s=
+github.com/coreruleset/ftw-tests-schema v1.1.0/go.mod h1:gRd9wBxjUI85HypWRDxJzbk1JqHC4KTxl0l/Y2p9QK4=
 github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -304,6 +306,7 @@ github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JT
 github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
 github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
 github.com/tonglil/buflogr v1.0.1 h1:WXFZLKxLfqcVSmckwiMCF8jJwjIgmStJmg63YKRF1p0=
+github.com/tonglil/buflogr v1.0.1/go.mod h1:yYWwvSpn/3uAaqjf6mJg/XMiAciaR0QcRJH2gJGDxNE=
 github.com/yargevad/filepathx v1.0.0 h1:SYcT+N3tYGi+NvazubCNlvgIPbzAk7i7y2dwg3I5FYc=
 github.com/yargevad/filepathx v1.0.0/go.mod h1:BprfX/gpYNJHJfc35GjRRpVcwWXS89gGulUIU5tK3tA=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

runner/run.go

@@ -12,6 +12,7 @@ import (
 	"github.com/google/uuid"
 	"github.com/rs/zerolog/log"
 
+	"github.com/coreruleset/ftw-tests-schema/types"
 	"github.com/coreruleset/go-ftw/check"
 	"github.com/coreruleset/go-ftw/config"
 	"github.com/coreruleset/go-ftw/ftwhttp"
@@ -99,7 +100,7 @@ func RunTest(runContext *TestRunContext, ftwTest test.FTWTest) error {
 			if err != nil {
 				return err
 			}
-			if err := RunStage(runContext, ftwCheck, testCase, stage.Stage); err != nil {
+			if err := RunStage(runContext, ftwCheck, testCase, stage.SD); err != nil {
 				return err
 			}
 		}
@@ -113,11 +114,11 @@ func RunTest(runContext *TestRunContext, ftwTest test.FTWTest) error {
 // ftwCheck is the current check utility
 // testCase is the test case the stage belongs to
 // stage is the stage you want to run
-func RunStage(runContext *TestRunContext, ftwCheck *check.FTWCheck, testCase test.Test, stage test.Stage) error {
+func RunStage(runContext *TestRunContext, ftwCheck *check.FTWCheck, testCase types.Test, stage types.StageData) error {
 	stageStartTime := time.Now()
 	stageID := uuid.NewString()
 	// Apply global overrides initially
-	testInput := stage.Input
+	testInput := (test.Input)(stage.Input)
 	test.ApplyInputOverrides(&runContext.Config.TestOverride.Overrides, &testInput)
 	expectedOutput := stage.Output
 	expectErr := false
@@ -180,7 +181,7 @@ func RunStage(runContext *TestRunContext, ftwCheck *check.FTWCheck, testCase tes
 	}
 
 	// Set expected test output in check
-	ftwCheck.SetExpectTestOutput(&expectedOutput)
+	ftwCheck.SetExpectTestOutput((*test.Output)(&expectedOutput))
 
 	// now get the test result based on output
 	testResult := checkResult(ftwCheck, response, responseErr)

runner/run_input_override_test.go

@@ -175,7 +175,7 @@ func (s *inputOverrideTestSuite) TestSetHostFromDestAddr() {
 
 	s.NotNil(testInput.Headers, "Header map must exist after overriding `dest_addr`")
 
-	hostHeader := testInput.Headers.Get("Host")
+	hostHeader := testInput.GetHeaders().Get("Host")
 	s.NotEqual("", hostHeader, "Host header must be set after overriding `dest_addr`")
 	s.Equal(overrideHost, hostHeader, "Host header must be identical to `dest_addr` after overrding `dest_addr`")
 }
@@ -191,7 +191,7 @@ func (s *inputOverrideTestSuite) TestSetHostFromHostHeaderOverride() {
 
 	test.ApplyInputOverrides(&s.cfg.TestOverride.Overrides, &testInput)
 
-	hostHeader := testInput.Headers.Get("Host")
+	hostHeader := testInput.GetHeaders().Get("Host")
 	s.NotEqual("", hostHeader, "Host header must be set after overriding the `Host` header")
 	if hostHeader == overrideHostHeader {
 		s.Equal(overrideHostHeader, hostHeader, "Host header override must take precence over OverrideEmptyHostHeader")
@@ -213,7 +213,7 @@ func (s *inputOverrideTestSuite) TestSetHeaderOverridingExistingOne() {
 
 	test.ApplyInputOverrides(&s.cfg.TestOverride.Overrides, &testInput)
 
-	overriddenHeader := testInput.Headers.Get("unique_id")
+	overriddenHeader := testInput.GetHeaders().Get("unique_id")
 	s.NotEqual("", overriddenHeader, "unique_id header must be set after overriding it")
 	s.Equal(overrideHeaderValue, overriddenHeader, "Host header must be identical to overridden `Host` header.")
 }
@@ -231,7 +231,7 @@ func (s *inputOverrideTestSuite) TestApplyInputOverrides() {
 
 	test.ApplyInputOverrides(&s.cfg.TestOverride.Overrides, &testInput)
 
-	overriddenHeader := testInput.Headers.Get("unique_id")
+	overriddenHeader := testInput.GetHeaders().Get("unique_id")
 	s.NotEqual("", overriddenHeader, "unique_id header must be set after overriding it")
 	s.Equal(overrideHeaderValue, overriddenHeader, "Host header must be identical to overridden `Host` header.")
 }

test/defaults.go

@@ -6,9 +6,15 @@ package test
 import (
 	"encoding/base64"
 
+	schema "github.com/coreruleset/ftw-tests-schema/types"
+	"github.com/coreruleset/go-ftw/ftwhttp"
 	"github.com/coreruleset/go-ftw/utils"
 )
 
+type Input schema.Input
+type Output schema.Output
+type FTWTest schema.FTWTest
+
 // GetMethod returns the proper semantic when the field is empty
 func (i *Input) GetMethod() string {
 	if i.Method == nil {
@@ -57,6 +63,14 @@ func (i *Input) GetPort() int {
 	return *i.Port
 }
 
+// GetHeaders returns the headers wrapped in a ftwhttp.Header
+func (i *Input) GetHeaders() ftwhttp.Header {
+	if i.Headers == nil {
+		return ftwhttp.Header{}
+	}
+	return ftwhttp.Header(i.Headers)
+}
+
 // GetRawRequest returns the proper raw data, and error if there was none
 func (i *Input) GetRawRequest() ([]byte, error) {
 	if utils.IsNotEmpty(i.EncodedRequest) {
@@ -68,3 +82,11 @@ func (i *Input) GetRawRequest() ([]byte, error) {
 	}
 	return nil, nil
 }
+
+// GetAutocompleteHeaders returns the autocompleteHeaders value, defaults to true
+func (i *Input) GetAutocompleteHeaders() bool {
+	if i.AutocompleteHeaders == nil {
+		return true
+	}
+	return *i.AutocompleteHeaders
+}

test/types.go

@@ -4,29 +4,10 @@
 package test
 
 import (
+	"github.com/coreruleset/ftw-tests-schema/types"
 	"github.com/coreruleset/go-ftw/ftwhttp"
 )
 
-// Input represents the input request in a stage
-// The fields `Version`, `Method` and `URI` we want to explicitly know when they are set to ""
-
-type Input struct {
-	DestAddr   *string        `yaml:"dest_addr,omitempty"`
-	Port       *int           `yaml:"port,omitempty"`
-	Protocol   *string        `yaml:"protocol,omitempty"`
-	URI        *string        `yaml:"uri,omitempty"`
-	Version    *string        `yaml:"version,omitempty"`
-	Headers    ftwhttp.Header `yaml:"headers,omitempty"`
-	Method     *string        `yaml:"method,omitempty"`
-	Data       *string        `yaml:"data,omitempty"`
-	SaveCookie *bool          `yaml:"save_cookie,omitempty"`
-	// Deprecated: replaced with AutocompleteHeaders
-	StopMagic           *bool  `yaml:"stop_magic"`
-	AutocompleteHeaders *bool  `yaml:"autocomplete_headers"`
-	EncodedRequest      string `yaml:"encoded_request,omitempty"`
-	RAWRequest          string `yaml:"raw_request,omitempty"`
-}
-
 // Overrides represents the overridden inputs that have to be applied to tests
 type Overrides struct {
 	DestAddr   *string        `yaml:"dest_addr,omitempty" koanf:"dest_addr,omitempty"`
@@ -46,42 +27,6 @@ type Overrides struct {
 	OverrideEmptyHostHeader *bool   `yaml:"override_empty_host_header,omitempty" koanf:"override_empty_host_header,omitempty"`
 }
 
-// Output is the response expected from the test
-type Output struct {
-	Status           []int  `yaml:"status,flow,omitempty"`
-	ResponseContains string `yaml:"response_contains,omitempty"`
-	LogContains      string `yaml:"log_contains,omitempty"`
-	NoLogContains    string `yaml:"no_log_contains,omitempty"`
-	ExpectError      *bool  `yaml:"expect_error,omitempty"`
-}
-
-// Stage is an individual test stage
-type Stage struct {
-	Input  Input  `yaml:"input"`
-	Output Output `yaml:"output"`
-}
-
-// Test is an individual test
-type Test struct {
-	TestTitle       string `yaml:"test_title"`
-	TestDescription string `yaml:"desc,omitempty"`
-	Stages          []struct {
-		Stage Stage `yaml:"stage"`
-	} `yaml:"stages"`
-}
-
-// FTWTest is the base type used when unmarshaling
-type FTWTest struct {
-	FileName string
-	Meta     struct {
-		Author      string `yaml:"author,omitempty"`
-		Enabled     *bool  `yaml:"enabled,omitempty"`
-		Name        string `yaml:"name,omitempty"`
-		Description string `yaml:"description,omitempty"`
-	} `yaml:"meta"`
-	Tests []Test `yaml:"tests"`
-}
-
 // ApplyInputOverride will check if config had global overrides and write that into the test.
 func ApplyInputOverrides(overrides *Overrides, input *Input) {
 	applySimpleOverrides(overrides, input)
@@ -98,8 +43,10 @@ func applyDestAddrOverride(overrides *Overrides, input *Input) {
 		if input.Headers == nil {
 			input.Headers = ftwhttp.Header{}
 		}
-		if overrides.OverrideEmptyHostHeader != nil && *overrides.OverrideEmptyHostHeader && input.Headers.Get("Host") == "" {
-			input.Headers.Set("Host", *overrides.DestAddr)
+		if overrides.OverrideEmptyHostHeader != nil &&
+			*overrides.OverrideEmptyHostHeader &&
+			input.GetHeaders().Get("Host") == "" {
+			input.GetHeaders().Set("Host", *overrides.DestAddr)
 		}
 	}
 }
@@ -148,7 +95,7 @@ func applyHeadersOverride(overrides *Overrides, input *Input) {
 			input.Headers = ftwhttp.Header{}
 		}
 		for k, v := range overrides.Headers {
-			input.Headers.Set(k, v)
+			input.GetHeaders().Set(k, v)
 		}
 	}
 }
@@ -159,14 +106,14 @@ func postLoadTestFTWTest(ftwTest *FTWTest) {
 	}
 }
 
-func postLoadTest(test *Test) {
+func postLoadTest(test *types.Test) {
 	for index := range test.Stages {
-		postLoadStage(&test.Stages[index].Stage)
+		postLoadStage(&test.Stages[index].SD)
 	}
 }
 
-func postLoadStage(stage *Stage) {
-	postLoadInput(&stage.Input)
+func postLoadStage(stage *types.StageData) {
+	postLoadInput((*Input)(&stage.Input))
 }
 
 func postLoadInput(input *Input) {

test/types_test.go

@@ -177,7 +177,7 @@ func (s *typesTestSuite) TestAutocompleteHeadersDefault_StopMagicDefault() {
 	test, err := GetTestFromYaml([]byte(autocompleteHeadersDefaultYaml))
 	s.NoError(err, "Parsing YAML shouldn't fail")
 
-	input := test.Tests[0].Stages[0].Stage.Input
+	input := test.Tests[0].Stages[0].SD.Input
 	s.True(*input.AutocompleteHeaders)
 	s.False(*input.StopMagic)
 }
@@ -186,15 +186,15 @@ func (s *typesTestSuite) TestAutocompleteHeadersDefault_StopMagicTrue() {
 	test, err := GetTestFromYaml([]byte(autocompleteHeadersDefaultYaml))
 	s.NoError(err, "Parsing YAML shouldn't fail")
 
-	input := test.Tests[1].Stages[0].Stage.Input
+	input := test.Tests[1].Stages[0].SD.Input
 	s.False(*input.AutocompleteHeaders)
 	s.True(*input.StopMagic)
 }
 func (s *typesTestSuite) TestAutocompleteHeadersDefault_StopMagicFalse() {
 	test, err := GetTestFromYaml([]byte(autocompleteHeadersDefaultYaml))
 	s.NoError(err, "Parsing YAML shouldn't fail")
 
-	input := test.Tests[2].Stages[0].Stage.Input
+	input := test.Tests[2].Stages[0].SD.Input
 	s.True(*input.AutocompleteHeaders)
 	s.False(*input.StopMagic)
 }
@@ -203,7 +203,7 @@ func (s *typesTestSuite) TestAutocompleteHeadersFalse_StopMagicDefault() {
 	test, err := GetTestFromYaml([]byte(autocompleteHeadersFalseYaml))
 	s.NoError(err, "Parsing YAML shouldn't fail")
 
-	input := test.Tests[0].Stages[0].Stage.Input
+	input := test.Tests[0].Stages[0].SD.Input
 	s.False(*input.AutocompleteHeaders)
 	s.True(*input.StopMagic)
 }
@@ -212,7 +212,7 @@ func (s *typesTestSuite) TestAutocompleteHeadersFalse_StopMagicTrue() {
 	test, err := GetTestFromYaml([]byte(autocompleteHeadersFalseYaml))
 	s.NoError(err, "Parsing YAML shouldn't fail")
 
-	input := test.Tests[1].Stages[0].Stage.Input
+	input := test.Tests[1].Stages[0].SD.Input
 	s.False(*input.AutocompleteHeaders)
 	s.True(*input.StopMagic)
 }
@@ -221,7 +221,7 @@ func (s *typesTestSuite) TestAutocompleteHeadersFalse_StopMagicFalse() {
 	test, err := GetTestFromYaml([]byte(autocompleteHeadersFalseYaml))
 	s.NoError(err, "Parsing YAML shouldn't fail")
 
-	input := test.Tests[2].Stages[0].Stage.Input
+	input := test.Tests[2].Stages[0].SD.Input
 	s.False(*input.AutocompleteHeaders)
 	s.True(*input.StopMagic)
 }
@@ -230,7 +230,7 @@ func (s *typesTestSuite) TestAutocompleteHeadersTrue_StopMagicDefault() {
 	test, err := GetTestFromYaml([]byte(autocompleteHeadersTrueYaml))
 	s.NoError(err, "Parsing YAML shouldn't fail")
 
-	input := test.Tests[0].Stages[0].Stage.Input
+	input := test.Tests[0].Stages[0].SD.Input
 	s.True(*input.AutocompleteHeaders)
 	s.False(*input.StopMagic)
 }
@@ -239,7 +239,7 @@ func (s *typesTestSuite) TestAutocompleteHeadersTrue_StopMagicTrue() {
 	test, err := GetTestFromYaml([]byte(autocompleteHeadersTrueYaml))
 	s.NoError(err, "Parsing YAML shouldn't fail")
 
-	input := test.Tests[1].Stages[0].Stage.Input
+	input := test.Tests[1].Stages[0].SD.Input
 	s.True(*input.AutocompleteHeaders)
 	s.False(*input.StopMagic)
 }
@@ -248,7 +248,7 @@ func (s *typesTestSuite) TestAutocompleteHeadersTrue_StopMagicFalse() {
 	test, err := GetTestFromYaml([]byte(autocompleteHeadersTrueYaml))
 	s.NoError(err, "Parsing YAML shouldn't fail")
 
-	input := test.Tests[2].Stages[0].Stage.Input
+	input := test.Tests[2].Stages[0].SD.Input
 	s.True(*input.AutocompleteHeaders)
 	s.False(*input.StopMagic)
 }