Skip to content

Commit

Permalink
wip: working post processing logic
Browse files Browse the repository at this point in the history
  • Loading branch information
@theseion
theseion committed Jun 11, 2023
1 parent 02f1a51 commit edf26fa
Show file tree
Hide file tree
Showing 5 changed files with 149 additions and 55 deletions.
57 changes: 26 additions & 31 deletions config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,17 +27,12 @@ func NewDefaultConfig() *FTWConfiguration {
// NewConfigFromFile reads configuration information from the config file if it exists,
// or uses `.ftw.yaml` as default file
func NewConfigFromFile(cfgFile string) (*FTWConfiguration, error) {
// Global koanf instance. Use "." as the key path delimiter. This can be "/" or any character.
var k = koanf.New(".")
var err error
cfg := NewDefaultConfig()

// first check if we had an explicit call with config file
if cfgFile == "" {
cfgFile = ".ftw.yaml"
}

_, err = os.Stat(cfgFile)
_, err := os.Stat(cfgFile)
if err != nil { // file does not exist, so we try the home folder

var home string
Expand All @@ -56,56 +51,39 @@ func NewConfigFromFile(cfgFile string) (*FTWConfiguration, error) {
return nil, err
}

k := getKoanfInstance()
err = k.Load(file.Provider(cfgFile), yaml.Parser())
if err != nil {
return nil, err
}

// At this point we have loaded our config, now we need to
// unmarshal the whole root module
err = k.UnmarshalWithConf("", cfg, koanf.UnmarshalConf{Tag: "koanf"})
if err != nil {
return nil, err
}

return cfg, err
return unmarshal(k)
}

// NewConfigFromEnv reads configuration information from environment variables that start with `FTW_`
func NewConfigFromEnv() (*FTWConfiguration, error) {
var err error
var k = koanf.New(".")
cfg := NewDefaultConfig()

err = k.Load(env.Provider("FTW_", ".", func(s string) string {
k := getKoanfInstance()
err := k.Load(env.Provider("FTW_", ".", func(s string) string {
return strings.ReplaceAll(strings.ToLower(
strings.TrimPrefix(s, "FTW_")), "_", ".")
}), nil)

if err != nil {
return nil, err
}
// Unmarshal the whole root module
err = k.UnmarshalWithConf("", cfg, koanf.UnmarshalConf{Tag: "koanf"})

return cfg, err
return unmarshal(k)
}

// NewConfigFromString initializes the configuration from a yaml formatted string. Useful for testing.
func NewConfigFromString(conf string) (*FTWConfiguration, error) {
var k = koanf.New(".")
var err error
cfg := NewDefaultConfig()

err = k.Load(rawbytes.Provider([]byte(conf)), yaml.Parser())
k := getKoanfInstance()
err := k.Load(rawbytes.Provider([]byte(conf)), yaml.Parser())
if err != nil {
return nil, err
}

// Unmarshal the whole root module
err = k.UnmarshalWithConf("", cfg, koanf.UnmarshalConf{Tag: "koanf"})

return cfg, err
return unmarshal(k)
}

// WithLogfile changes the logfile in the configuration.
Expand Down Expand Up @@ -137,3 +115,20 @@ func (c *FTWConfiguration) WithMaxMarkerRetries(retries int) {
func (c *FTWConfiguration) WithMaxMarkerLogLines(retries int) {
c.MaxMarkerLogLines = retries
}

// Unmarshal the loaded koanf instance into a configuration object
func unmarshal(k *koanf.Koanf) (*FTWConfiguration, error) {
config := NewDefaultConfig()
err := k.UnmarshalWithConf("", config, koanf.UnmarshalConf{Tag: "koanf"})
if err != nil {
return nil, err
}

return config, nil
}

// Get the global koanf instance
func getKoanfInstance() *koanf.Koanf {
// Use "." as the key path delimiter. This can be "/" or any character.
return koanf.New(".")
}
11 changes: 10 additions & 1 deletion runner/run.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -424,8 +424,17 @@ func applyInputOverride(o config.FTWTestOverride, testRequest *test.Input) error
testRequest.Data = overrides.Data
}

// TODO: postprocess
if overrides.SaveCookie != nil {
testRequest.SaveCookie = overrides.SaveCookie
}

if overrides.StopMagic != nil {
testRequest.StopMagic = *overrides.StopMagic
testRequest.StopMagic = overrides.StopMagic
}

if overrides.NoAutocompleteHeaders != nil {
testRequest.NoAutocompleteHeaders = overrides.NoAutocompleteHeaders
}

if overrides.EncodedRequest != nil {
Expand Down
87 changes: 86 additions & 1 deletion test/files.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,13 @@ package test

import (
"errors"
"fmt"
"os"
"regexp"
"strings"

"github.com/goccy/go-yaml"
"github.com/goccy/go-yaml/ast"
"github.com/rs/zerolog/log"
"github.com/yargevad/filepathx"
)
Expand Down Expand Up @@ -56,11 +59,13 @@ func GetTestFromYaml(testYaml []byte) (ftwTest FTWTest, err error) {
return FTWTest{}, err
}

postProcess(testYaml, &ftwTest)

return ftwTest, nil
}

func readTestYaml(testYaml []byte) (t FTWTest, err error) {
err = yaml.Unmarshal([]byte(testYaml), &t)
err = yaml.Unmarshal(testYaml, &t)
return t, err
}

Expand Down Expand Up @@ -105,3 +110,83 @@ func describeYamlError(yamlError error) string {

return "We do not have an extended explanation of this error."
}

// TODO: make more general (env, string)
// TODO: also post process overrides
func postProcess(testYaml []byte, ftwTest *FTWTest) error {
yamlString := string(testYaml)
for index, test := range ftwTest.Tests {
path, err := yaml.PathString(fmt.Sprintf("$.tests[%d]", index))
if err != nil {
return err
}
node, err := path.ReadNode(strings.NewReader(yamlString))
if err != nil {
return err
}
err = postProcessTest(node.(*ast.MappingNode), &test)
if err != nil {
return err
}
}
return nil
}

func postProcessTest(node *ast.MappingNode, test *Test) error {
nodeString := node.String()
for index, stage := range test.Stages {
path, err := yaml.PathString(fmt.Sprintf("$.stages[%d]", index))
if err != nil {
return err
}
stageNode, err := path.ReadNode(strings.NewReader(nodeString))
if err != nil {
return err
}
err = postProcessStage(stageNode.(*ast.MappingValueNode), &stage.Stage)
if err != nil {
return err
}
}
return nil
}

func postProcessStage(node *ast.MappingValueNode, stage *Stage) error {
return postProcessInput(node.Value.(*ast.MappingNode), &stage.Input)
}

func postProcessInput(node *ast.MappingNode, input *Input) error {
return postProcessNoAutocompleteHeaders(node, input)
}

func postProcessNoAutocompleteHeaders(node ast.Node, input *Input) error {
noAutocompleteHeadersMissing := false
stopMagicMissing := false
err := readField(node, "no_autocomplete_headers", &input.NoAutocompleteHeaders)
if err != nil {
noAutocompleteHeadersMissing = true
}
err = readField(node, "stop_magic", &input.StopMagic)
if err != nil {
stopMagicMissing = true
}

if noAutocompleteHeadersMissing && stopMagicMissing {
return nil
}
if noAutocompleteHeadersMissing && !stopMagicMissing {
input.NoAutocompleteHeaders = input.StopMagic
return nil
}
input.StopMagic = input.NoAutocompleteHeaders
return nil
}

func readField(node ast.Node, fieldName string, out interface{}) error {
path, err := yaml.PathString("$." + fieldName)
if err != nil {
return err
}
err = path.Read(strings.NewReader(node.String()), out)
return err
}
1 change: 1 addition & 0 deletions test/files_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ var yamlTest = `
-
stage:
input:
no_autocomplete_headers: false
dest_addr: "127.0.0.1"
port: 80
headers:
Expand Down
48 changes: 26 additions & 22 deletions test/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,32 +6,36 @@ import "github.com/coreruleset/go-ftw/ftwhttp"
// The fields `Version`, `Method` and `URI` we want to explicitly know when they are set to ""

type Input struct {
DestAddr *string `yaml:"dest_addr,omitempty" koanf:"dest_addr,omitempty"`
Port *int `yaml:"port,omitempty" koanf:"port,omitempty"`
Protocol *string `yaml:"protocol,omitempty" koanf:"protocol,omitempty"`
URI *string `yaml:"uri,omitempty" koanf:"uri,omitempty"`
Version *string `yaml:"version,omitempty" koanf:"version,omitempty"`
Headers ftwhttp.Header `yaml:"headers,omitempty" koanf:"headers,omitempty"`
Method *string `yaml:"method,omitempty" koanf:"method,omitempty"`
Data *string `yaml:"data,omitempty" koanf:"data,omitempty"`
SaveCookie bool `yaml:"save_cookie,omitempty" koanf:"save_cookie,omitempty"`
NoAutocompleteHeaders bool `yaml:"no_autocomplete_headers" koanf:"no_autocomplete_headers,omitempty"`
EncodedRequest string `yaml:"encoded_request,omitempty" koanf:"encoded_request,omitempty"`
RAWRequest string `yaml:"raw_request,omitempty" koanf:"raw_request,omitempty"`
DestAddr *string `yaml:"dest_addr,omitempty" koanf:"dest_addr,omitempty"`
Port *int `yaml:"port,omitempty" koanf:"port,omitempty"`
Protocol *string `yaml:"protocol,omitempty" koanf:"protocol,omitempty"`
URI *string `yaml:"uri,omitempty" koanf:"uri,omitempty"`
Version *string `yaml:"version,omitempty" koanf:"version,omitempty"`
Headers ftwhttp.Header `yaml:"headers,omitempty" koanf:"headers,omitempty"`
Method *string `yaml:"method,omitempty" koanf:"method,omitempty"`
Data *string `yaml:"data,omitempty" koanf:"data,omitempty"`
SaveCookie bool `yaml:"save_cookie,omitempty" koanf:"save_cookie,omitempty"`
// Deprecated, replaced with NoAutocompleteHeaders
StopMagic bool `yaml:"stop_magic" koanf:"stop_magic,omitempty"`
NoAutocompleteHeaders bool `yaml:"no_autocomplete_headers" koanf:"no_autocomplete_headers,omitempty"`
EncodedRequest string `yaml:"encoded_request,omitempty" koanf:"encoded_request,omitempty"`
RAWRequest string `yaml:"raw_request,omitempty" koanf:"raw_request,omitempty"`
}

// Overrides represents the overridden inputs that have to be applied to tests
type Overrides struct {
DestAddr *string `yaml:"dest_addr,omitempty" koanf:"dest_addr,omitempty"`
Port *int `yaml:"port,omitempty" koanf:"port,omitempty"`
Protocol *string `yaml:"protocol,omitempty" koanf:"protocol,omitempty"`
URI *string `yaml:"uri,omitempty" koanf:"uri,omitempty"`
Version *string `yaml:"version,omitempty" koanf:"version,omitempty"`
Headers ftwhttp.Header `yaml:"headers,omitempty" koanf:"headers,omitempty"`
Method *string `yaml:"method,omitempty" koanf:"method,omitempty"`
Data *string `yaml:"data,omitempty" koanf:"data,omitempty"`
//SaveCookie bool `yaml:"save_cookie,omitempty" koanf:"save_cookie,omitempty"`
// NoAutocompleteHeaders bool `yaml:"no_autocomplete_headers" koanf:"no_autocomplete_headers,omitempty"`
DestAddr *string `yaml:"dest_addr,omitempty" koanf:"dest_addr,omitempty"`
Port *int `yaml:"port,omitempty" koanf:"port,omitempty"`
Protocol *string `yaml:"protocol,omitempty" koanf:"protocol,omitempty"`
URI *string `yaml:"uri,omitempty" koanf:"uri,omitempty"`
Version *string `yaml:"version,omitempty" koanf:"version,omitempty"`
Headers ftwhttp.Header `yaml:"headers,omitempty" koanf:"headers,omitempty"`
Method *string `yaml:"method,omitempty" koanf:"method,omitempty"`
Data *string `yaml:"data,omitempty" koanf:"data,omitempty"`
SaveCookie bool `yaml:"save_cookie,omitempty" koanf:"save_cookie,omitempty"`
// Deprecated, replaced with NoAutocompleteHeaders
StopMagic bool `yaml:"stop_magic" koanf:"stop_magic,omitempty"`
NoAutocompleteHeaders bool `yaml:"no_autocomplete_headers" koanf:"no_autocomplete_headers,omitempty"`
EncodedRequest *string `yaml:"encoded_request,omitempty" koanf:"encoded_request,omitempty"`
RAWRequest *string `yaml:"raw_request,omitempty" koanf:"raw_request,omitempty"`
OverrideEmptyHostHeader bool `yaml:"override_empty_host_header,omitempty" koanf:"override_empty_host_header,omitempty"`
Expand Down

0 comments on commit edf26fa

Please sign in to comment.