From fb41287b7f401e4d7b4d44aec81b88f31bfb953e Mon Sep 17 00:00:00 2001 From: Matteo Pace Date: Sat, 3 Aug 2024 17:43:33 +0200 Subject: [PATCH 01/10] wip: crs 4.5, albedo --- testing/coreruleset/.ftw.yml | 16 ++- testing/coreruleset/albedo_test.go | 124 ++++++++++++++++++++++++ testing/coreruleset/coreruleset_test.go | 41 +++----- testing/coreruleset/go.mod | 10 +- testing/coreruleset/go.sum | 20 ++-- 5 files changed, 164 insertions(+), 47 deletions(-) create mode 100644 testing/coreruleset/albedo_test.go diff --git a/testing/coreruleset/.ftw.yml b/testing/coreruleset/.ftw.yml index 0cc93009b..c2932f4b1 100644 --- a/testing/coreruleset/.ftw.yml +++ b/testing/coreruleset/.ftw.yml @@ -1,13 +1,23 @@ --- testoverride: ignore: - 920100-4: 'Invalid uri, Coraza not reached - 404 page not found' 920100-5: 'Invalid uri, Coraza not reached - 404 page not found' 920100-8: 'Go/http allows a colon in the path. Test expects status 400 or 403 (Apache behaviour)' 920270-4: 'Rule works, log contains 920270. Test expects status 400 (Apache behaviour)' - 920272-5: 'Rule works, log contains 920272. Test expects status 400 (Apache behaviour)' 920290-1: 'Rule works, log contains 920290. Test expects status 400 (Apache behaviour)' 920290-4: 'Go/http returns 400 Bad Request: missing required Host header' 920430-8: 'Go/http does not allow HTTP/3.0 - 505 HTTP Version Not Supported' - 932200-13: 'wip' 930110-7: 'CRS issue: https://github.com/coreruleset/coreruleset/issues/3736' + + # TODO: investigate + 932200-13: 'Failing only in multiphase evalution' + 932300-10: 'Failing only in multiphase evalution' + 933120-2: 'Failing only in multiphase evalution' + 920274-1: '' + 920280-3: '' + 920430-3: '' + 920430-5: '' + 920430-9: '' + 920610-2: '' + 920620-1: '' + diff --git a/testing/coreruleset/albedo_test.go b/testing/coreruleset/albedo_test.go new file mode 100644 index 000000000..e4ff4daca --- /dev/null +++ b/testing/coreruleset/albedo_test.go @@ -0,0 +1,124 @@ +// Copyright 2024 Juan Pablo Tosso and the OWASP Coraza contributors +// SPDX-License-Identifier: Apache-2.0 + +// These benchmarks don't currently compile with TinyGo +//go:build !tinygo +// +build !tinygo + +// Note: The following code has been extracted from https://github.com/coreruleset/albedo/blob/main/server/server.go +// TODO: Make it possible to import albedo. +package coreruleset + +import ( + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "log" + "net/http" + "strings" + "testing" +) + +type reflectionSpec struct { + Status int `json:"status"` + Headers map[string]string `json:"headers"` + Body string `json:"body"` + EncodedBody string `json:"encodedBody"` + LogMessage string `json:"logMessage"` +} + +func handleReflect(t testing.TB, w http.ResponseWriter, r *http.Request) { + log.Println("Received reflection request") + + body, err := io.ReadAll(r.Body) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + _, err = w.Write([]byte("Failed to parse request body")) + if err != nil { + log.Printf("Failed to write response body: %s", err.Error()) + } + log.Println("Failed to parse request body") + return + } + spec := &reflectionSpec{} + if err = json.Unmarshal(body, spec); err != nil { + w.WriteHeader(http.StatusBadRequest) + _, err = w.Write([]byte("Invalid JSON in request body")) + if err != nil { + log.Printf("Failed to write response body: %s", err.Error()) + } + log.Println("Invalid JSON in request body") + return + } + + if spec.LogMessage != "" { + log.Println(spec.LogMessage) + } + + for name, value := range spec.Headers { + log.Printf("Reflecting header '%s':'%s'", name, value) + w.Header().Add(name, value) + } + + if spec.Status > 0 && spec.Status < 100 || spec.Status >= 600 { + w.WriteHeader(http.StatusBadRequest) + _, err = w.Write([]byte(fmt.Sprintf("Invalid status code: %d", spec.Status))) + if err != nil { + log.Printf("Failed to write response body: %s", err.Error()) + } + log.Printf("Invalid status code: %d", spec.Status) + return + } + status := spec.Status + if status == 0 { + status = http.StatusOK + } + log.Printf("Reflecting status '%d'", status) + w.WriteHeader(status) + + responseBody, err := decodeBody(t, spec) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + _, err = w.Write([]byte(err.Error())) + if err != nil { + log.Printf("Failed to write response body: %s", err.Error()) + } + log.Println(err.Error()) + return + } + + if responseBody == "" { + return + } + + responseBodyBytes := []byte(responseBody) + if len(responseBody) > 200 { + responseBody = responseBody[:min(len(responseBody), 200)] + "..." + } + log.Printf("Reflecting body '%s'", responseBody) + _, err = w.Write(responseBodyBytes) + if err != nil { + log.Printf("Failed to write response body: %s", err.Error()) + } +} + +func decodeBody(t testing.TB, spec *reflectionSpec) (string, error) { + t.Helper() + if spec.Body != "" { + return spec.Body, nil + } + + if spec.EncodedBody == "" { + return "", nil + } + + decoder := base64.NewDecoder(base64.StdEncoding, strings.NewReader(spec.EncodedBody)) + bodyBytes, err := io.ReadAll(decoder) + if err != nil { + return "", errors.New("invalid base64 encoding of response body") + + } + return string(bodyBytes), nil +} diff --git a/testing/coreruleset/coreruleset_test.go b/testing/coreruleset/coreruleset_test.go index f926bda2c..eb9249c23 100644 --- a/testing/coreruleset/coreruleset_test.go +++ b/testing/coreruleset/coreruleset_test.go @@ -221,31 +221,12 @@ SecRule REQUEST_HEADERS:X-CRS-Test "@rx ^.*$" \ } s := httptest.NewServer(txhttp.WrapHandler(waf, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Emulates https://github.com/coreruleset/albedo behavior defer r.Body.Close() w.Header().Set("Content-Type", "text/plain") switch { - case r.URL.Path == "/anything", r.URL.Path == "/post": - body, err := io.ReadAll(r.Body) - // Emulated httpbin behaviour: /anything and /post endpoints act as an echo server, writing back the request body - if r.Header.Get("Content-Type") == "application/x-www-form-urlencoded" { - // Tests 954120-1 and 954120-2 are the only two calling /anything with a POST and payload is urlencoded - if err != nil { - t.Fatalf("handler can not read request body: %v", err) - } - urldecodedBody, err := url.QueryUnescape(string(body)) - if err != nil { - t.Logf("[warning] handler can not unescape urlencoded request body: %v", err) - // If the body can't be unescaped, we will keep going with the received body - urldecodedBody = string(body) - } - fmt.Fprint(w, urldecodedBody) - } else { - _, err = w.Write(body) - if err != nil { - t.Fatalf("handler can not write request body: %v", err) - } - } - + case r.URL.Path == "/reflect": + handleReflect(t, w, r) case strings.HasPrefix(r.URL.Path, "/base64/"): // Emulated httpbin behaviour: /base64 endpoint write the decoded base64 into the response body b64Decoded, err := b64.StdEncoding.DecodeString(strings.TrimPrefix(r.URL.Path, "/base64/")) @@ -254,8 +235,7 @@ SecRule REQUEST_HEADERS:X-CRS-Test "@rx ^.*$" \ } fmt.Fprint(w, string(b64Decoded)) default: - // Common path "/status/200" defaults here - fmt.Fprint(w, "Hello!") + // Albedo return 200 with no body } }))) defer s.Close() @@ -266,7 +246,7 @@ SecRule REQUEST_HEADERS:X-CRS-Test "@rx ^.*$" \ if err != nil { return err } - ftwt, err := test.GetTestFromYaml(yaml) + ftwt, err := test.GetTestFromYaml(yaml, path) if err != nil { return err } @@ -292,16 +272,19 @@ SecRule REQUEST_HEADERS:X-CRS-Test "@rx ^.*$" \ cfg.WithLogfile(errorPath) cfg.TestOverride.Overrides.DestAddr = &host cfg.TestOverride.Overrides.Port = &port - res, err := runner.Run(cfg, tests, runner.RunnerConfig{ ShowTime: false, }, output.NewOutput("quiet", os.Stdout)) if err != nil { t.Fatal(err) } - - if len(res.Stats.Failed) > 0 { - t.Errorf("failed tests: %v", res.Stats.Failed) + totalIgnored := len(res.Stats.Ignored) + if totalIgnored > 0 { + t.Logf("[info] %d ignored tests: %v", totalIgnored, res.Stats.Ignored) + } + totalFailed := len(res.Stats.Failed) + if totalFailed > 0 { + t.Errorf("[fatal] %d failed tests: %v", totalFailed, res.Stats.Failed) } } diff --git a/testing/coreruleset/go.mod b/testing/coreruleset/go.mod index a6ccd641c..ebdc8461b 100644 --- a/testing/coreruleset/go.mod +++ b/testing/coreruleset/go.mod @@ -4,9 +4,9 @@ go 1.22 require ( github.com/bmatcuk/doublestar/v4 v4.6.1 - github.com/corazawaf/coraza-coreruleset/v4 v4.3.0 + github.com/corazawaf/coraza-coreruleset/v4 v4.5.0 github.com/corazawaf/coraza/v3 v3.0.0-00010101000000-000000000000 - github.com/coreruleset/go-ftw v0.6.4 + github.com/coreruleset/go-ftw v1.0.3 github.com/rs/zerolog v1.33.0 ) @@ -15,8 +15,8 @@ require ( github.com/Masterminds/semver v1.5.0 // indirect github.com/Masterminds/sprig v2.22.0+incompatible // indirect github.com/corazawaf/libinjection-go v0.2.1 // indirect - github.com/coreruleset/ftw-tests-schema v1.1.0 // indirect - github.com/fatih/color v1.16.0 // indirect + github.com/coreruleset/ftw-tests-schema/v2 v2.1.0 // indirect + github.com/fatih/color v1.17.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 // indirect github.com/goccy/go-yaml v1.11.3 // indirect @@ -28,7 +28,7 @@ require ( github.com/knadh/koanf/maps v0.1.1 // indirect github.com/knadh/koanf/parsers/yaml v0.1.0 // indirect github.com/knadh/koanf/providers/env v0.1.0 // indirect - github.com/knadh/koanf/providers/file v0.1.0 // indirect + github.com/knadh/koanf/providers/file v1.1.0 // indirect github.com/knadh/koanf/providers/rawbytes v0.1.0 // indirect github.com/knadh/koanf/v2 v2.1.1 // indirect github.com/kyokomi/emoji/v2 v2.2.13 // indirect diff --git a/testing/coreruleset/go.sum b/testing/coreruleset/go.sum index da62e37e0..f409eae9f 100644 --- a/testing/coreruleset/go.sum +++ b/testing/coreruleset/go.sum @@ -6,19 +6,19 @@ github.com/Masterminds/sprig v2.22.0+incompatible h1:z4yfnGrZ7netVz+0EDJ0Wi+5VZC github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= github.com/bmatcuk/doublestar/v4 v4.6.1 h1:FH9SifrbvJhnlQpztAx++wlkk70QBf0iBWDwNy7PA4I= github.com/bmatcuk/doublestar/v4 v4.6.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= -github.com/corazawaf/coraza-coreruleset/v4 v4.3.0 h1:izzVRUxfvVf1OXhRQXpFm1jj3g/cIlLu9SiNzXOW7XU= -github.com/corazawaf/coraza-coreruleset/v4 v4.3.0/go.mod h1:RQMGurig+irQq7v21yq7rM/9SAEf1bT6hCSplJ0ByKY= +github.com/corazawaf/coraza-coreruleset/v4 v4.5.0 h1:4BDr9/yWKSJ7Ch3h7SvSqJBASju73+EqIIF0WxjsFgQ= +github.com/corazawaf/coraza-coreruleset/v4 v4.5.0/go.mod h1:1FQt1p+JSQ6tYrafMqZrEEdDmhq6aVuIJdnk+bM9hMY= github.com/corazawaf/libinjection-go v0.2.1 h1:vNJ7L6c4xkhRgYU6sIO0Tl54TmeCQv/yfxBma30Dy/Y= github.com/corazawaf/libinjection-go v0.2.1/go.mod h1:OP4TM7xdJ2skyXqNX1AN1wN5nNZEmJNuWbNPOItn7aw= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/coreruleset/ftw-tests-schema v1.1.0 h1:3+NYrdLE3HVmOc3nGrisRBBvY9lGjePUrV+YkT5Ay3s= -github.com/coreruleset/ftw-tests-schema v1.1.0/go.mod h1:gRd9wBxjUI85HypWRDxJzbk1JqHC4KTxl0l/Y2p9QK4= -github.com/coreruleset/go-ftw v0.6.4 h1:EdDNld38Jv4lxqHS+csGOJuHu1/8rpp4TlrFyoijTPk= -github.com/coreruleset/go-ftw v0.6.4/go.mod h1:IayMjfOmmNNBcqTcZU92e6UZTy79/eFdmJEmRu8tLs4= +github.com/coreruleset/ftw-tests-schema/v2 v2.1.0 h1:2ilKzKRG5UzzxBcrJLXFtPalStdQ9jzzaYFuFk0OEk0= +github.com/coreruleset/ftw-tests-schema/v2 v2.1.0/go.mod h1:ZHVFX5ses4+5IxUP0ufCNg/VqRWxziH6ZuUca092Hxo= +github.com/coreruleset/go-ftw v1.0.3 h1:DVqoTvBGXtAd0knvlyxr7d/bk1xMkzq8cfSbjxeOxLI= +github.com/coreruleset/go-ftw v1.0.3/go.mod h1:NzYL8DIoAbulVVneBlnudFdlJziO3mJU3iIV2fZK46U= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= -github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= +github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= +github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= github.com/foxcpp/go-mockdns v1.1.0 h1:jI0rD8M0wuYAxL7r/ynTrCQQq0BVqfB99Vgk7DlmewI= github.com/foxcpp/go-mockdns v1.1.0/go.mod h1:IhLeSFGed3mJIAXPH2aiRQB+kqz7oqu8ld2qVbOu7Wk= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= @@ -52,8 +52,8 @@ github.com/knadh/koanf/parsers/yaml v0.1.0 h1:ZZ8/iGfRLvKSaMEECEBPM1HQslrZADk8fP github.com/knadh/koanf/parsers/yaml v0.1.0/go.mod h1:cvbUDC7AL23pImuQP0oRw/hPuccrNBS2bps8asS0CwY= github.com/knadh/koanf/providers/env v0.1.0 h1:LqKteXqfOWyx5Ab9VfGHmjY9BvRXi+clwyZozgVRiKg= github.com/knadh/koanf/providers/env v0.1.0/go.mod h1:RE8K9GbACJkeEnkl8L/Qcj8p4ZyPXZIQ191HJi44ZaQ= -github.com/knadh/koanf/providers/file v0.1.0 h1:fs6U7nrV58d3CFAFh8VTde8TM262ObYf3ODrc//Lp+c= -github.com/knadh/koanf/providers/file v0.1.0/go.mod h1:rjJ/nHQl64iYCtAW2QQnF0eSmDEX/YZ/eNFj5yR6BvA= +github.com/knadh/koanf/providers/file v1.1.0 h1:MTjA+gRrVl1zqgetEAIaXHqYje0XSosxSiMD4/7kz0o= +github.com/knadh/koanf/providers/file v1.1.0/go.mod h1:/faSBcv2mxPVjFrXck95qeoyoZ5myJ6uxN8OOVNJJCI= github.com/knadh/koanf/providers/rawbytes v0.1.0 h1:dpzgu2KO6uf6oCb4aP05KDmKmAmI51k5pe8RYKQ0qME= github.com/knadh/koanf/providers/rawbytes v0.1.0/go.mod h1:mMTB1/IcJ/yE++A2iEZbY1MLygX7vttU+C+S/YmPu9c= github.com/knadh/koanf/v2 v2.1.1 h1:/R8eXqasSTsmDCsAyYj+81Wteg8AqrV9CP6gvsTsOmM= From b949c78c14675c51a434fce34d95f67f775216b7 Mon Sep 17 00:00:00 2001 From: Matteo Pace Date: Fri, 9 Aug 2024 10:50:00 +0200 Subject: [PATCH 02/10] wip --- testing/coreruleset/.ftw.yml | 7 +++---- testing/coreruleset/coreruleset_test.go | 5 +++++ testing/coreruleset/go.mod | 3 ++- testing/coreruleset/go.sum | 6 ++++-- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/testing/coreruleset/.ftw.yml b/testing/coreruleset/.ftw.yml index c2932f4b1..0a7e07148 100644 --- a/testing/coreruleset/.ftw.yml +++ b/testing/coreruleset/.ftw.yml @@ -13,11 +13,10 @@ testoverride: 932200-13: 'Failing only in multiphase evalution' 932300-10: 'Failing only in multiphase evalution' 933120-2: 'Failing only in multiphase evalution' - 920274-1: '' + 920274-1: 'Host validation. Apache expects status 400, Coraza should trigger the rule 920274. Run and check it.' 920280-3: '' 920430-3: '' 920430-5: '' 920430-9: '' - 920610-2: '' - 920620-1: '' - + 920610-2: 'fragments, Coraza might just happly accept them. Run and check it.' + 920620-1: 'Rule checks if multiple Content-Type headers are kepts. Go/http might keep them and trigger the rule. Run and check it.' diff --git a/testing/coreruleset/coreruleset_test.go b/testing/coreruleset/coreruleset_test.go index eb9249c23..83e4b3e3c 100644 --- a/testing/coreruleset/coreruleset_test.go +++ b/testing/coreruleset/coreruleset_test.go @@ -272,6 +272,11 @@ SecRule REQUEST_HEADERS:X-CRS-Test "@rx ^.*$" \ cfg.WithLogfile(errorPath) cfg.TestOverride.Overrides.DestAddr = &host cfg.TestOverride.Overrides.Port = &port + + // TODO(M4tteoP) + // cfg.LoadPlatformOverrides(): .ftw.yml should become a platform override file + // Tests would not just be ignored, but new expectations would be set for the specific platform + // E.g. see https://github.com/coreruleset/coreruleset/blob/main/tests/regression/nginx-overrides.yaml res, err := runner.Run(cfg, tests, runner.RunnerConfig{ ShowTime: false, }, output.NewOutput("quiet", os.Stdout)) diff --git a/testing/coreruleset/go.mod b/testing/coreruleset/go.mod index ebdc8461b..4a242e732 100644 --- a/testing/coreruleset/go.mod +++ b/testing/coreruleset/go.mod @@ -6,7 +6,7 @@ require ( github.com/bmatcuk/doublestar/v4 v4.6.1 github.com/corazawaf/coraza-coreruleset/v4 v4.5.0 github.com/corazawaf/coraza/v3 v3.0.0-00010101000000-000000000000 - github.com/coreruleset/go-ftw v1.0.3 + github.com/coreruleset/go-ftw v1.0.4-0.20240809050408-f8169f0325ac github.com/rs/zerolog v1.33.0 ) @@ -48,6 +48,7 @@ require ( golang.org/x/net v0.28.0 // indirect golang.org/x/sync v0.8.0 // indirect golang.org/x/sys v0.23.0 // indirect + golang.org/x/time v0.6.0 // indirect golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect rsc.io/binaryregexp v0.2.0 // indirect diff --git a/testing/coreruleset/go.sum b/testing/coreruleset/go.sum index f409eae9f..41985ba33 100644 --- a/testing/coreruleset/go.sum +++ b/testing/coreruleset/go.sum @@ -13,8 +13,8 @@ github.com/corazawaf/libinjection-go v0.2.1/go.mod h1:OP4TM7xdJ2skyXqNX1AN1wN5nN github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreruleset/ftw-tests-schema/v2 v2.1.0 h1:2ilKzKRG5UzzxBcrJLXFtPalStdQ9jzzaYFuFk0OEk0= github.com/coreruleset/ftw-tests-schema/v2 v2.1.0/go.mod h1:ZHVFX5ses4+5IxUP0ufCNg/VqRWxziH6ZuUca092Hxo= -github.com/coreruleset/go-ftw v1.0.3 h1:DVqoTvBGXtAd0knvlyxr7d/bk1xMkzq8cfSbjxeOxLI= -github.com/coreruleset/go-ftw v1.0.3/go.mod h1:NzYL8DIoAbulVVneBlnudFdlJziO3mJU3iIV2fZK46U= +github.com/coreruleset/go-ftw v1.0.4-0.20240809050408-f8169f0325ac h1:I++204ogJDnOyYQrMs6IdfTYRBIMr4A9Dtix+XdtZEc= +github.com/coreruleset/go-ftw v1.0.4-0.20240809050408-f8169f0325ac/go.mod h1:gI2N2EYdTIZnXQbsdzBRxbj/zSaYEyrhLJUCOJ3VK6I= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= @@ -111,6 +111,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= +golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= From dba20fd8a9fa2454214ed54e7a85b4387724e2da Mon Sep 17 00:00:00 2001 From: Matteo Pace Date: Fri, 9 Aug 2024 11:22:47 +0200 Subject: [PATCH 03/10] test timeout --- testing/coreruleset/coreruleset_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/testing/coreruleset/coreruleset_test.go b/testing/coreruleset/coreruleset_test.go index 83e4b3e3c..925a5bb34 100644 --- a/testing/coreruleset/coreruleset_test.go +++ b/testing/coreruleset/coreruleset_test.go @@ -21,6 +21,7 @@ import ( "strconv" "strings" "testing" + "time" "github.com/bmatcuk/doublestar/v4" "github.com/coreruleset/go-ftw/config" @@ -278,7 +279,8 @@ SecRule REQUEST_HEADERS:X-CRS-Test "@rx ^.*$" \ // Tests would not just be ignored, but new expectations would be set for the specific platform // E.g. see https://github.com/coreruleset/coreruleset/blob/main/tests/regression/nginx-overrides.yaml res, err := runner.Run(cfg, tests, runner.RunnerConfig{ - ShowTime: false, + ShowTime: false, + ConnectTimeout: 10 * time.Second, // WIP: Defaults to 3s but looks to be not enough }, output.NewOutput("quiet", os.Stdout)) if err != nil { t.Fatal(err) From d68bfba9cdf184d3facb279caf2953e14bf1436a Mon Sep 17 00:00:00 2001 From: Matteo Pace Date: Fri, 9 Aug 2024 13:08:45 +0200 Subject: [PATCH 04/10] Fix timeout to make CI work --- testing/coreruleset/albedo_test.go | 1 + testing/coreruleset/coreruleset_test.go | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/testing/coreruleset/albedo_test.go b/testing/coreruleset/albedo_test.go index e4ff4daca..980660616 100644 --- a/testing/coreruleset/albedo_test.go +++ b/testing/coreruleset/albedo_test.go @@ -30,6 +30,7 @@ type reflectionSpec struct { } func handleReflect(t testing.TB, w http.ResponseWriter, r *http.Request) { + t.Helper() log.Println("Received reflection request") body, err := io.ReadAll(r.Body) diff --git a/testing/coreruleset/coreruleset_test.go b/testing/coreruleset/coreruleset_test.go index 925a5bb34..8b06dae91 100644 --- a/testing/coreruleset/coreruleset_test.go +++ b/testing/coreruleset/coreruleset_test.go @@ -279,8 +279,8 @@ SecRule REQUEST_HEADERS:X-CRS-Test "@rx ^.*$" \ // Tests would not just be ignored, but new expectations would be set for the specific platform // E.g. see https://github.com/coreruleset/coreruleset/blob/main/tests/regression/nginx-overrides.yaml res, err := runner.Run(cfg, tests, runner.RunnerConfig{ - ShowTime: false, - ConnectTimeout: 10 * time.Second, // WIP: Defaults to 3s but looks to be not enough + ShowTime: false, + ReadTimeout: 3 * time.Second, // Defaults to 1s but looks to be not enough in the CI }, output.NewOutput("quiet", os.Stdout)) if err != nil { t.Fatal(err) From dbaf2f7e052503fa97d331a8f573dfda76410b4a Mon Sep 17 00:00:00 2001 From: Matteo Pace Date: Fri, 9 Aug 2024 23:05:57 +0200 Subject: [PATCH 05/10] wip moving to overrides --- testing/coreruleset/.ftw-overrides.yml | 54 +++++++++++++++++++++++++ testing/coreruleset/.ftw.yml | 5 +-- testing/coreruleset/coreruleset_test.go | 2 +- 3 files changed, 56 insertions(+), 5 deletions(-) create mode 100644 testing/coreruleset/.ftw-overrides.yml diff --git a/testing/coreruleset/.ftw-overrides.yml b/testing/coreruleset/.ftw-overrides.yml new file mode 100644 index 000000000..9eefcbb87 --- /dev/null +++ b/testing/coreruleset/.ftw-overrides.yml @@ -0,0 +1,54 @@ +version: "v0.0.0" +meta: + engine: "coraza" + platform: "go" + annotations: + - purpose: "Overrides for CRS test suite running against Coraza deployed as a Go middleware" +test_overrides: + # - rule_id: 920100 + # test_ids: [5] + # reason: "Invalid uri, Coraza not reached - 404 page not found" + # output: + # log: + # expect_ids: [920100] + - rule_id: 920100 + test_ids: [8] + reason: | + On Apache is not allowed to put a colon in the path. Go/http allows it. + Note that the colon is a legal character in a regular path segment, according to the RFC. + output: + status: 200 + log: + no_expect_ids: [920100] + - rule_id: 920270 + test_ids: [4] + reason: "Rule works, Apache test expects status 400" + output: + log: + expect_ids: [920270] + - rule_id: 920274 + test_ids: [1] + reason: "Host validation. Apache expects status 400. Coraza correctly triggers the rule 920274" + output: + log: + expect_ids: [920274] + # - rule_id: 920290 + # test_ids: [1] + # reason: "Rule works, Apache test expects status 400" + # output: + # log: + # expect_ids: [920290] + # - rule_id: 920290 + # test_ids: [4] + # reason: "Go/http returns 400 Bad Request: missing required Host header" + # output: + # status: 400 + # log: + # no_expect_ids: [920290] + - rule_id: 920430 + test_ids: [8] + reason: "Go/http does not allow HTTP/3.0 - 505 HTTP Version Not Supported" + output: + status: 505 + log: + no_expect_ids: [920430] diff --git a/testing/coreruleset/.ftw.yml b/testing/coreruleset/.ftw.yml index 0a7e07148..9ad846d6d 100644 --- a/testing/coreruleset/.ftw.yml +++ b/testing/coreruleset/.ftw.yml @@ -2,18 +2,15 @@ testoverride: ignore: 920100-5: 'Invalid uri, Coraza not reached - 404 page not found' - 920100-8: 'Go/http allows a colon in the path. Test expects status 400 or 403 (Apache behaviour)' - 920270-4: 'Rule works, log contains 920270. Test expects status 400 (Apache behaviour)' 920290-1: 'Rule works, log contains 920290. Test expects status 400 (Apache behaviour)' 920290-4: 'Go/http returns 400 Bad Request: missing required Host header' - 920430-8: 'Go/http does not allow HTTP/3.0 - 505 HTTP Version Not Supported' 930110-7: 'CRS issue: https://github.com/coreruleset/coreruleset/issues/3736' # TODO: investigate 932200-13: 'Failing only in multiphase evalution' 932300-10: 'Failing only in multiphase evalution' 933120-2: 'Failing only in multiphase evalution' - 920274-1: 'Host validation. Apache expects status 400, Coraza should trigger the rule 920274. Run and check it.' + # 920274-1: 'Host validation. Apache expects status 400, Coraza should trigger the rule 920274. Run and check it.' 920280-3: '' 920430-3: '' 920430-5: '' diff --git a/testing/coreruleset/coreruleset_test.go b/testing/coreruleset/coreruleset_test.go index 8b06dae91..5c08cc5fb 100644 --- a/testing/coreruleset/coreruleset_test.go +++ b/testing/coreruleset/coreruleset_test.go @@ -273,11 +273,11 @@ SecRule REQUEST_HEADERS:X-CRS-Test "@rx ^.*$" \ cfg.WithLogfile(errorPath) cfg.TestOverride.Overrides.DestAddr = &host cfg.TestOverride.Overrides.Port = &port - // TODO(M4tteoP) // cfg.LoadPlatformOverrides(): .ftw.yml should become a platform override file // Tests would not just be ignored, but new expectations would be set for the specific platform // E.g. see https://github.com/coreruleset/coreruleset/blob/main/tests/regression/nginx-overrides.yaml + cfg.LoadPlatformOverrides(".ftw-overrides.yml") res, err := runner.Run(cfg, tests, runner.RunnerConfig{ ShowTime: false, ReadTimeout: 3 * time.Second, // Defaults to 1s but looks to be not enough in the CI From 85f01f9622d5e12463c1a18b52981e039eac9dd0 Mon Sep 17 00:00:00 2001 From: Matteo Pace Date: Mon, 12 Aug 2024 10:32:11 +0200 Subject: [PATCH 06/10] removes rule added to overrides --- testing/coreruleset/.ftw.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/testing/coreruleset/.ftw.yml b/testing/coreruleset/.ftw.yml index 9ad846d6d..788acd14b 100644 --- a/testing/coreruleset/.ftw.yml +++ b/testing/coreruleset/.ftw.yml @@ -10,7 +10,6 @@ testoverride: 932200-13: 'Failing only in multiphase evalution' 932300-10: 'Failing only in multiphase evalution' 933120-2: 'Failing only in multiphase evalution' - # 920274-1: 'Host validation. Apache expects status 400, Coraza should trigger the rule 920274. Run and check it.' 920280-3: '' 920430-3: '' 920430-5: '' From a1203eabbd06d06a7541a935945e8036d2c8cbdf Mon Sep 17 00:00:00 2001 From: Matteo Pace Date: Sun, 22 Sep 2024 13:02:58 +0200 Subject: [PATCH 07/10] rebase --- examples/http-server/go.mod | 2 +- examples/http-server/go.sum | 4 ++-- go.sum | 2 -- testing/coreruleset/go.mod | 4 ++-- testing/coreruleset/go.sum | 9 +++++---- 5 files changed, 10 insertions(+), 11 deletions(-) diff --git a/examples/http-server/go.mod b/examples/http-server/go.mod index fde9666a8..34b92f6d3 100644 --- a/examples/http-server/go.mod +++ b/examples/http-server/go.mod @@ -8,7 +8,7 @@ require ( github.com/corazawaf/libinjection-go v0.2.1 // indirect github.com/magefile/mage v1.15.0 // indirect github.com/petar-dambovaliev/aho-corasick v0.0.0-20240411101913-e07a1f0e8eb4 // indirect - github.com/tidwall/gjson v1.17.1 // indirect + github.com/tidwall/gjson v1.17.3 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect golang.org/x/net v0.28.0 // indirect diff --git a/examples/http-server/go.sum b/examples/http-server/go.sum index 9be2d9cf1..a53d2325a 100644 --- a/examples/http-server/go.sum +++ b/examples/http-server/go.sum @@ -10,8 +10,8 @@ github.com/miekg/dns v1.1.57 h1:Jzi7ApEIzwEPLHWRcafCN9LZSBbqQpxjt/wpgvg7wcM= github.com/miekg/dns v1.1.57/go.mod h1:uqRjCRUuEAA6qsOiJvDd+CFo/vW+y5WR6SNmHE55hZk= github.com/petar-dambovaliev/aho-corasick v0.0.0-20240411101913-e07a1f0e8eb4 h1:1Kw2vDBXmjop+LclnzCb/fFy+sgb3gYARwfmoUcQe6o= github.com/petar-dambovaliev/aho-corasick v0.0.0-20240411101913-e07a1f0e8eb4/go.mod h1:EHPiTAKtiFmrMldLUNswFwfZ2eJIYBHktdaUTZxYWRw= -github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U= -github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.17.3 h1:bwWLZU7icoKRG+C+0PNwIKC6FCJO/Q3p2pZvuP0jN94= +github.com/tidwall/gjson v1.17.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= diff --git a/go.sum b/go.sum index c4593576c..ea15a8c43 100644 --- a/go.sum +++ b/go.sum @@ -14,8 +14,6 @@ github.com/miekg/dns v1.1.57 h1:Jzi7ApEIzwEPLHWRcafCN9LZSBbqQpxjt/wpgvg7wcM= github.com/miekg/dns v1.1.57/go.mod h1:uqRjCRUuEAA6qsOiJvDd+CFo/vW+y5WR6SNmHE55hZk= github.com/petar-dambovaliev/aho-corasick v0.0.0-20240411101913-e07a1f0e8eb4 h1:1Kw2vDBXmjop+LclnzCb/fFy+sgb3gYARwfmoUcQe6o= github.com/petar-dambovaliev/aho-corasick v0.0.0-20240411101913-e07a1f0e8eb4/go.mod h1:EHPiTAKtiFmrMldLUNswFwfZ2eJIYBHktdaUTZxYWRw= -github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U= -github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.17.3 h1:bwWLZU7icoKRG+C+0PNwIKC6FCJO/Q3p2pZvuP0jN94= github.com/tidwall/gjson v1.17.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= diff --git a/testing/coreruleset/go.mod b/testing/coreruleset/go.mod index 4a242e732..6b8930c33 100644 --- a/testing/coreruleset/go.mod +++ b/testing/coreruleset/go.mod @@ -38,8 +38,7 @@ require ( github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/petar-dambovaliev/aho-corasick v0.0.0-20240411101913-e07a1f0e8eb4 // indirect - github.com/stretchr/testify v1.9.0 // indirect - github.com/tidwall/gjson v1.17.1 // indirect + github.com/tidwall/gjson v1.17.3 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect github.com/valllabh/ocsf-schema-golang v1.0.3 // indirect @@ -50,6 +49,7 @@ require ( golang.org/x/sys v0.23.0 // indirect golang.org/x/time v0.6.0 // indirect golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect + google.golang.org/protobuf v1.34.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect rsc.io/binaryregexp v0.2.0 // indirect ) diff --git a/testing/coreruleset/go.sum b/testing/coreruleset/go.sum index 41985ba33..5c29d11bb 100644 --- a/testing/coreruleset/go.sum +++ b/testing/coreruleset/go.sum @@ -84,11 +84,10 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8= github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U= -github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.17.3 h1:bwWLZU7icoKRG+C+0PNwIKC6FCJO/Q3p2pZvuP0jN94= +github.com/tidwall/gjson v1.17.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= @@ -117,6 +116,8 @@ golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJ golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= From 9247c8df577a5984b486c3fc3a7edbc947e9765d Mon Sep 17 00:00:00 2001 From: Matteo Pace Date: Sun, 22 Sep 2024 15:59:21 +0200 Subject: [PATCH 08/10] some progress --- testing/coreruleset/.ftw-overrides.yml | 30 ++++++++++++-------------- testing/coreruleset/.ftw.yml | 2 -- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/testing/coreruleset/.ftw-overrides.yml b/testing/coreruleset/.ftw-overrides.yml index 9eefcbb87..0d32f24d4 100644 --- a/testing/coreruleset/.ftw-overrides.yml +++ b/testing/coreruleset/.ftw-overrides.yml @@ -7,10 +7,9 @@ meta: test_overrides: # - rule_id: 920100 # test_ids: [5] - # reason: "Invalid uri, Coraza not reached - 404 page not found" + # reason: "Invalid uri, Coraza not reached - 301 returned" # output: - # log: - # expect_ids: [920100] + # status: 301 - rule_id: 920100 test_ids: [8] reason: | @@ -32,19 +31,18 @@ test_overrides: output: log: expect_ids: [920274] - # - rule_id: 920290 - # test_ids: [1] - # reason: "Rule works, Apache test expects status 400" - # output: - # log: - # expect_ids: [920290] - # - rule_id: 920290 - # test_ids: [4] - # reason: "Go/http returns 400 Bad Request: missing required Host header" - # output: - # status: 400 - # log: - # no_expect_ids: [920290] + - rule_id: 920290 + test_ids: [1] + reason: "TODO" + output: + log: + expect_ids: [920280] # TODO: understand why 920280 (Missing Host Header) is triggered and not 920290 (Empty Host header). See what go-ftw sends. + - rule_id: 920290 + test_ids: [4] + reason: "TODO" + output: + log: + expect_ids: [920280] # TODO: understand why 920280 (Missing Host Header) is triggered and not 920290 (Empty Host header). See what go-ftw sends. - rule_id: 920430 test_ids: [8] reason: "Go/http does not allow HTTP/3.0 - 505 HTTP Version Not Supported" diff --git a/testing/coreruleset/.ftw.yml b/testing/coreruleset/.ftw.yml index 788acd14b..0430651da 100644 --- a/testing/coreruleset/.ftw.yml +++ b/testing/coreruleset/.ftw.yml @@ -2,8 +2,6 @@ testoverride: ignore: 920100-5: 'Invalid uri, Coraza not reached - 404 page not found' - 920290-1: 'Rule works, log contains 920290. Test expects status 400 (Apache behaviour)' - 920290-4: 'Go/http returns 400 Bad Request: missing required Host header' 930110-7: 'CRS issue: https://github.com/coreruleset/coreruleset/issues/3736' # TODO: investigate From 4c10bf9a8163f0a42e8b79a5a43c8707fd44673c Mon Sep 17 00:00:00 2001 From: Matteo Pace Date: Tue, 24 Sep 2024 23:49:55 +0200 Subject: [PATCH 09/10] uses albedo as a library --- examples/http-server/go.mod | 1 + examples/http-server/go.sum | 8 +- go.mod | 6 +- go.sum | 8 +- go.work | 2 +- testing/coreruleset/albedo_test.go | 125 ------------------------ testing/coreruleset/coreruleset_test.go | 13 +-- testing/coreruleset/go.mod | 5 +- testing/coreruleset/go.sum | 28 ++++-- 9 files changed, 39 insertions(+), 157 deletions(-) delete mode 100644 testing/coreruleset/albedo_test.go diff --git a/examples/http-server/go.mod b/examples/http-server/go.mod index 34b92f6d3..dbc7cabda 100644 --- a/examples/http-server/go.mod +++ b/examples/http-server/go.mod @@ -13,5 +13,6 @@ require ( github.com/tidwall/pretty v1.2.1 // indirect golang.org/x/net v0.28.0 // indirect golang.org/x/sync v0.8.0 // indirect + golang.org/x/tools v0.22.0 // indirect rsc.io/binaryregexp v0.2.0 // indirect ) diff --git a/examples/http-server/go.sum b/examples/http-server/go.sum index a53d2325a..b9d5b6e58 100644 --- a/examples/http-server/go.sum +++ b/examples/http-server/go.sum @@ -17,15 +17,15 @@ github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JT github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= -golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= +golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= +golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/go.mod b/go.mod index fcb8f4753..8f0a8e297 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/corazawaf/coraza/v3 -go 1.22 +go 1.22.3 // Testing dependencies: // - go-mockdns @@ -34,8 +34,8 @@ require ( github.com/miekg/dns v1.1.57 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect - golang.org/x/mod v0.17.0 // indirect + golang.org/x/mod v0.18.0 // indirect golang.org/x/sys v0.23.0 // indirect - golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect + golang.org/x/tools v0.22.0 // indirect google.golang.org/protobuf v1.34.1 // indirect ) diff --git a/go.sum b/go.sum index ea15a8c43..80b0a15c7 100644 --- a/go.sum +++ b/go.sum @@ -33,8 +33,8 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= +golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= @@ -85,8 +85,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.15.0/go.mod h1:hpksKq4dtpQWS1uQ61JkdqWM3LscIS6Slf+VVkm+wQk= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= +golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= diff --git a/go.work b/go.work index 6d1cff10f..6907c6bf4 100644 --- a/go.work +++ b/go.work @@ -1,4 +1,4 @@ -go 1.22 +go 1.22.3 use ( . diff --git a/testing/coreruleset/albedo_test.go b/testing/coreruleset/albedo_test.go deleted file mode 100644 index 980660616..000000000 --- a/testing/coreruleset/albedo_test.go +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright 2024 Juan Pablo Tosso and the OWASP Coraza contributors -// SPDX-License-Identifier: Apache-2.0 - -// These benchmarks don't currently compile with TinyGo -//go:build !tinygo -// +build !tinygo - -// Note: The following code has been extracted from https://github.com/coreruleset/albedo/blob/main/server/server.go -// TODO: Make it possible to import albedo. -package coreruleset - -import ( - "encoding/base64" - "encoding/json" - "errors" - "fmt" - "io" - "log" - "net/http" - "strings" - "testing" -) - -type reflectionSpec struct { - Status int `json:"status"` - Headers map[string]string `json:"headers"` - Body string `json:"body"` - EncodedBody string `json:"encodedBody"` - LogMessage string `json:"logMessage"` -} - -func handleReflect(t testing.TB, w http.ResponseWriter, r *http.Request) { - t.Helper() - log.Println("Received reflection request") - - body, err := io.ReadAll(r.Body) - if err != nil { - w.WriteHeader(http.StatusBadRequest) - _, err = w.Write([]byte("Failed to parse request body")) - if err != nil { - log.Printf("Failed to write response body: %s", err.Error()) - } - log.Println("Failed to parse request body") - return - } - spec := &reflectionSpec{} - if err = json.Unmarshal(body, spec); err != nil { - w.WriteHeader(http.StatusBadRequest) - _, err = w.Write([]byte("Invalid JSON in request body")) - if err != nil { - log.Printf("Failed to write response body: %s", err.Error()) - } - log.Println("Invalid JSON in request body") - return - } - - if spec.LogMessage != "" { - log.Println(spec.LogMessage) - } - - for name, value := range spec.Headers { - log.Printf("Reflecting header '%s':'%s'", name, value) - w.Header().Add(name, value) - } - - if spec.Status > 0 && spec.Status < 100 || spec.Status >= 600 { - w.WriteHeader(http.StatusBadRequest) - _, err = w.Write([]byte(fmt.Sprintf("Invalid status code: %d", spec.Status))) - if err != nil { - log.Printf("Failed to write response body: %s", err.Error()) - } - log.Printf("Invalid status code: %d", spec.Status) - return - } - status := spec.Status - if status == 0 { - status = http.StatusOK - } - log.Printf("Reflecting status '%d'", status) - w.WriteHeader(status) - - responseBody, err := decodeBody(t, spec) - if err != nil { - w.WriteHeader(http.StatusBadRequest) - _, err = w.Write([]byte(err.Error())) - if err != nil { - log.Printf("Failed to write response body: %s", err.Error()) - } - log.Println(err.Error()) - return - } - - if responseBody == "" { - return - } - - responseBodyBytes := []byte(responseBody) - if len(responseBody) > 200 { - responseBody = responseBody[:min(len(responseBody), 200)] + "..." - } - log.Printf("Reflecting body '%s'", responseBody) - _, err = w.Write(responseBodyBytes) - if err != nil { - log.Printf("Failed to write response body: %s", err.Error()) - } -} - -func decodeBody(t testing.TB, spec *reflectionSpec) (string, error) { - t.Helper() - if spec.Body != "" { - return spec.Body, nil - } - - if spec.EncodedBody == "" { - return "", nil - } - - decoder := base64.NewDecoder(base64.StdEncoding, strings.NewReader(spec.EncodedBody)) - bodyBytes, err := io.ReadAll(decoder) - if err != nil { - return "", errors.New("invalid base64 encoding of response body") - - } - return string(bodyBytes), nil -} diff --git a/testing/coreruleset/coreruleset_test.go b/testing/coreruleset/coreruleset_test.go index 5c08cc5fb..3529d35a9 100644 --- a/testing/coreruleset/coreruleset_test.go +++ b/testing/coreruleset/coreruleset_test.go @@ -9,7 +9,6 @@ package coreruleset import ( "bufio" - b64 "encoding/base64" "fmt" "io" "io/fs" @@ -35,6 +34,7 @@ import ( "github.com/corazawaf/coraza/v3" txhttp "github.com/corazawaf/coraza/v3/http" "github.com/corazawaf/coraza/v3/types" + albedo "github.com/coreruleset/albedo/server" ) func BenchmarkCRSCompilation(b *testing.B) { @@ -222,19 +222,12 @@ SecRule REQUEST_HEADERS:X-CRS-Test "@rx ^.*$" \ } s := httptest.NewServer(txhttp.WrapHandler(waf, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // Emulates https://github.com/coreruleset/albedo behavior + // CRS regression tests are expected to be run with https://github.com/coreruleset/albedo as backend server defer r.Body.Close() w.Header().Set("Content-Type", "text/plain") switch { case r.URL.Path == "/reflect": - handleReflect(t, w, r) - case strings.HasPrefix(r.URL.Path, "/base64/"): - // Emulated httpbin behaviour: /base64 endpoint write the decoded base64 into the response body - b64Decoded, err := b64.StdEncoding.DecodeString(strings.TrimPrefix(r.URL.Path, "/base64/")) - if err != nil { - t.Fatalf("handler can not decode base64: %v", err) - } - fmt.Fprint(w, string(b64Decoded)) + albedo.Handler().ServeHTTP(w, r) default: // Albedo return 200 with no body } diff --git a/testing/coreruleset/go.mod b/testing/coreruleset/go.mod index 6b8930c33..1d49532c6 100644 --- a/testing/coreruleset/go.mod +++ b/testing/coreruleset/go.mod @@ -1,11 +1,12 @@ module github.com/corazawaf/coraza/v3/testing/coreruleset -go 1.22 +go 1.22.3 require ( github.com/bmatcuk/doublestar/v4 v4.6.1 github.com/corazawaf/coraza-coreruleset/v4 v4.5.0 github.com/corazawaf/coraza/v3 v3.0.0-00010101000000-000000000000 + github.com/coreruleset/albedo v0.0.16-0.20240924185852-4b95a321ebfd github.com/coreruleset/go-ftw v1.0.4-0.20240809050408-f8169f0325ac github.com/rs/zerolog v1.33.0 ) @@ -31,6 +32,7 @@ require ( github.com/knadh/koanf/providers/file v1.1.0 // indirect github.com/knadh/koanf/providers/rawbytes v0.1.0 // indirect github.com/knadh/koanf/v2 v2.1.1 // indirect + github.com/kr/text v0.2.0 // indirect github.com/kyokomi/emoji/v2 v2.2.13 // indirect github.com/magefile/mage v1.15.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect @@ -38,6 +40,7 @@ require ( github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/petar-dambovaliev/aho-corasick v0.0.0-20240411101913-e07a1f0e8eb4 // indirect + github.com/rogpeppe/go-internal v1.13.1 // indirect github.com/tidwall/gjson v1.17.3 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect diff --git a/testing/coreruleset/go.sum b/testing/coreruleset/go.sum index 5c29d11bb..89a629afa 100644 --- a/testing/coreruleset/go.sum +++ b/testing/coreruleset/go.sum @@ -11,12 +11,15 @@ github.com/corazawaf/coraza-coreruleset/v4 v4.5.0/go.mod h1:1FQt1p+JSQ6tYrafMqZr github.com/corazawaf/libinjection-go v0.2.1 h1:vNJ7L6c4xkhRgYU6sIO0Tl54TmeCQv/yfxBma30Dy/Y= github.com/corazawaf/libinjection-go v0.2.1/go.mod h1:OP4TM7xdJ2skyXqNX1AN1wN5nNZEmJNuWbNPOItn7aw= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreruleset/albedo v0.0.16-0.20240924185852-4b95a321ebfd h1:4KjOGOv4I81ZOuW/TY29oe0yZCNeNGuPt8HXvmjukPI= +github.com/coreruleset/albedo v0.0.16-0.20240924185852-4b95a321ebfd/go.mod h1:6mYBASfvvRM2ckXgYO7N5nyKAj8OqLnT4+YLbM0/XWE= github.com/coreruleset/ftw-tests-schema/v2 v2.1.0 h1:2ilKzKRG5UzzxBcrJLXFtPalStdQ9jzzaYFuFk0OEk0= github.com/coreruleset/ftw-tests-schema/v2 v2.1.0/go.mod h1:ZHVFX5ses4+5IxUP0ufCNg/VqRWxziH6ZuUca092Hxo= github.com/coreruleset/go-ftw v1.0.4-0.20240809050408-f8169f0325ac h1:I++204ogJDnOyYQrMs6IdfTYRBIMr4A9Dtix+XdtZEc= github.com/coreruleset/go-ftw v1.0.4-0.20240809050408-f8169f0325ac/go.mod h1:gI2N2EYdTIZnXQbsdzBRxbj/zSaYEyrhLJUCOJ3VK6I= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= github.com/foxcpp/go-mockdns v1.1.0 h1:jI0rD8M0wuYAxL7r/ynTrCQQq0BVqfB99Vgk7DlmewI= @@ -58,6 +61,10 @@ github.com/knadh/koanf/providers/rawbytes v0.1.0 h1:dpzgu2KO6uf6oCb4aP05KDmKmAmI github.com/knadh/koanf/providers/rawbytes v0.1.0/go.mod h1:mMTB1/IcJ/yE++A2iEZbY1MLygX7vttU+C+S/YmPu9c= github.com/knadh/koanf/v2 v2.1.1 h1:/R8eXqasSTsmDCsAyYj+81Wteg8AqrV9CP6gvsTsOmM= github.com/knadh/koanf/v2 v2.1.1/go.mod h1:4mnTRbZCK+ALuBXHZMjDfG9y714L7TykVnZkXbMU3Es= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kyokomi/emoji/v2 v2.2.13 h1:GhTfQa67venUUvmleTNFnb+bi7S3aocF7ZCXU9fSO7U= github.com/kyokomi/emoji/v2 v2.2.13/go.mod h1:JUcn42DTdsXJo1SWanHh4HKDEyPaR5CqkmoirZZP9qE= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= @@ -79,8 +86,10 @@ github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx github.com/petar-dambovaliev/aho-corasick v0.0.0-20240411101913-e07a1f0e8eb4 h1:1Kw2vDBXmjop+LclnzCb/fFy+sgb3gYARwfmoUcQe6o= github.com/petar-dambovaliev/aho-corasick v0.0.0-20240411101913-e07a1f0e8eb4/go.mod h1:EHPiTAKtiFmrMldLUNswFwfZ2eJIYBHktdaUTZxYWRw= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8= github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= @@ -99,8 +108,8 @@ github.com/yargevad/filepathx v1.0.0 h1:SYcT+N3tYGi+NvazubCNlvgIPbzAk7i7y2dwg3I5 github.com/yargevad/filepathx v1.0.0/go.mod h1:BprfX/gpYNJHJfc35GjRRpVcwWXS89gGulUIU5tK3tA= golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= -golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= +golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= @@ -112,14 +121,15 @@ golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= +golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= From c6af3d446bc42856580a861f1551fdaa125d7050 Mon Sep 17 00:00:00 2001 From: Matteo Pace Date: Wed, 25 Sep 2024 00:07:48 +0200 Subject: [PATCH 10/10] finalizes some notes and comments, deps --- examples/http-server/go.mod | 2 +- examples/http-server/go.sum | 8 ++++---- testing/coreruleset/.ftw.yml | 3 +++ testing/coreruleset/coreruleset_test.go | 16 +++++----------- testing/coreruleset/go.mod | 6 +++--- testing/coreruleset/go.sum | 12 ++++++------ 6 files changed, 22 insertions(+), 25 deletions(-) diff --git a/examples/http-server/go.mod b/examples/http-server/go.mod index dbc7cabda..11ab0d944 100644 --- a/examples/http-server/go.mod +++ b/examples/http-server/go.mod @@ -11,7 +11,7 @@ require ( github.com/tidwall/gjson v1.17.3 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect - golang.org/x/net v0.28.0 // indirect + golang.org/x/net v0.29.0 // indirect golang.org/x/sync v0.8.0 // indirect golang.org/x/tools v0.22.0 // indirect rsc.io/binaryregexp v0.2.0 // indirect diff --git a/examples/http-server/go.sum b/examples/http-server/go.sum index b9d5b6e58..bde8489e6 100644 --- a/examples/http-server/go.sum +++ b/examples/http-server/go.sum @@ -19,12 +19,12 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= -golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= -golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= diff --git a/testing/coreruleset/.ftw.yml b/testing/coreruleset/.ftw.yml index 0430651da..23077a4c2 100644 --- a/testing/coreruleset/.ftw.yml +++ b/testing/coreruleset/.ftw.yml @@ -1,6 +1,9 @@ +# Tests should not just be ignored via .ftw.yml, but new expectations for each test should be set. +# Avoid as much as possible adding new entries here, in favor of .ftw-overrides.yml --- testoverride: ignore: + 920100-4: 'Invalid uri, Coraza not reached - 404 page not found' 920100-5: 'Invalid uri, Coraza not reached - 404 page not found' 930110-7: 'CRS issue: https://github.com/coreruleset/coreruleset/issues/3736' diff --git a/testing/coreruleset/coreruleset_test.go b/testing/coreruleset/coreruleset_test.go index 3529d35a9..6fe6ead86 100644 --- a/testing/coreruleset/coreruleset_test.go +++ b/testing/coreruleset/coreruleset_test.go @@ -221,16 +221,13 @@ SecRule REQUEST_HEADERS:X-CRS-Test "@rx ^.*$" \ t.Fatal(err) } + // CRS regression tests are expected to be run with https://github.com/coreruleset/albedo as backend server s := httptest.NewServer(txhttp.WrapHandler(waf, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // CRS regression tests are expected to be run with https://github.com/coreruleset/albedo as backend server defer r.Body.Close() + // TODO: Investigate why we need to enforce text/plain to have response body tests working. + // Check the Content-Type set by albed and SecResponseBodyMimeType w.Header().Set("Content-Type", "text/plain") - switch { - case r.URL.Path == "/reflect": - albedo.Handler().ServeHTTP(w, r) - default: - // Albedo return 200 with no body - } + albedo.Handler().ServeHTTP(w, r) }))) defer s.Close() @@ -266,10 +263,7 @@ SecRule REQUEST_HEADERS:X-CRS-Test "@rx ^.*$" \ cfg.WithLogfile(errorPath) cfg.TestOverride.Overrides.DestAddr = &host cfg.TestOverride.Overrides.Port = &port - // TODO(M4tteoP) - // cfg.LoadPlatformOverrides(): .ftw.yml should become a platform override file - // Tests would not just be ignored, but new expectations would be set for the specific platform - // E.g. see https://github.com/coreruleset/coreruleset/blob/main/tests/regression/nginx-overrides.yaml + cfg.LoadPlatformOverrides(".ftw-overrides.yml") res, err := runner.Run(cfg, tests, runner.RunnerConfig{ ShowTime: false, diff --git a/testing/coreruleset/go.mod b/testing/coreruleset/go.mod index 1d49532c6..f9bfde462 100644 --- a/testing/coreruleset/go.mod +++ b/testing/coreruleset/go.mod @@ -46,10 +46,10 @@ require ( github.com/tidwall/pretty v1.2.1 // indirect github.com/valllabh/ocsf-schema-golang v1.0.3 // indirect github.com/yargevad/filepathx v1.0.0 // indirect - golang.org/x/crypto v0.26.0 // indirect - golang.org/x/net v0.28.0 // indirect + golang.org/x/crypto v0.27.0 // indirect + golang.org/x/net v0.29.0 // indirect golang.org/x/sync v0.8.0 // indirect - golang.org/x/sys v0.23.0 // indirect + golang.org/x/sys v0.25.0 // indirect golang.org/x/time v0.6.0 // indirect golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect google.golang.org/protobuf v1.34.1 // indirect diff --git a/testing/coreruleset/go.sum b/testing/coreruleset/go.sum index 89a629afa..27d8a14ce 100644 --- a/testing/coreruleset/go.sum +++ b/testing/coreruleset/go.sum @@ -106,19 +106,19 @@ github.com/valllabh/ocsf-schema-golang v1.0.3 h1:eR8k/3jP/OOqB8LRCtdJ4U+vlgd/gk5 github.com/valllabh/ocsf-schema-golang v1.0.3/go.mod h1:sZ3as9xqm1SSK5feFWIR2CuGeGRhsM7TR1MbpBctzPk= github.com/yargevad/filepathx v1.0.0 h1:SYcT+N3tYGi+NvazubCNlvgIPbzAk7i7y2dwg3I5FYc= github.com/yargevad/filepathx v1.0.0/go.mod h1:BprfX/gpYNJHJfc35GjRRpVcwWXS89gGulUIU5tK3tA= -golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= -golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= +golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= +golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= -golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= -golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA=