Skip to content

Commit

Permalink
updates tests to CRS 4.5, albedo (#1122)
Browse files Browse the repository at this point in the history
* wip: crs 4.5, albedo

* wip

* test timeout

* Fix timeout to make CI work

* wip moving to overrides

* removes rule added to overrides

* rebase

* some progress

* uses albedo as a library

* finalizes some notes and comments, deps
  • Loading branch information
M4tteoP committed Sep 27, 2024
1 parent 5eb9b53 commit c531a97
Show file tree
Hide file tree
Showing 10 changed files with 160 additions and 107 deletions.
5 changes: 3 additions & 2 deletions examples/http-server/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,11 @@ require (
github.com/corazawaf/libinjection-go v0.2.1 // indirect
github.com/magefile/mage v1.15.0 // indirect
github.com/petar-dambovaliev/aho-corasick v0.0.0-20240411101913-e07a1f0e8eb4 // indirect
github.com/tidwall/gjson v1.17.1 // indirect
github.com/tidwall/gjson v1.17.3 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.1 // indirect
golang.org/x/net v0.28.0 // indirect
golang.org/x/net v0.29.0 // indirect
golang.org/x/sync v0.8.0 // indirect
golang.org/x/tools v0.22.0 // indirect
rsc.io/binaryregexp v0.2.0 // indirect
)
20 changes: 10 additions & 10 deletions examples/http-server/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -10,22 +10,22 @@ github.com/miekg/dns v1.1.57 h1:Jzi7ApEIzwEPLHWRcafCN9LZSBbqQpxjt/wpgvg7wcM=
github.com/miekg/dns v1.1.57/go.mod h1:uqRjCRUuEAA6qsOiJvDd+CFo/vW+y5WR6SNmHE55hZk=
github.com/petar-dambovaliev/aho-corasick v0.0.0-20240411101913-e07a1f0e8eb4 h1:1Kw2vDBXmjop+LclnzCb/fFy+sgb3gYARwfmoUcQe6o=
github.com/petar-dambovaliev/aho-corasick v0.0.0-20240411101913-e07a1f0e8eb4/go.mod h1:EHPiTAKtiFmrMldLUNswFwfZ2eJIYBHktdaUTZxYWRw=
github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U=
github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/gjson v1.17.3 h1:bwWLZU7icoKRG+C+0PNwIKC6FCJO/Q3p2pZvuP0jN94=
github.com/tidwall/gjson v1.17.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA=
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE=
golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg=
golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0=
golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo=
golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0=
golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM=
golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA=
golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c=
rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
6 changes: 3 additions & 3 deletions go.mod
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
module github.com/corazawaf/coraza/v3

go 1.22
go 1.22.3

// Testing dependencies:
// - go-mockdns
Expand Down Expand Up @@ -34,8 +34,8 @@ require (
github.com/miekg/dns v1.1.57 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.1 // indirect
golang.org/x/mod v0.17.0 // indirect
golang.org/x/mod v0.18.0 // indirect
golang.org/x/sys v0.25.0 // indirect
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
golang.org/x/tools v0.22.0 // indirect
google.golang.org/protobuf v1.34.1 // indirect
)
8 changes: 4 additions & 4 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,8 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA=
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0=
golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
Expand Down Expand Up @@ -85,8 +85,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
golang.org/x/tools v0.15.0/go.mod h1:hpksKq4dtpQWS1uQ61JkdqWM3LscIS6Slf+VVkm+wQk=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA=
golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=
google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
Expand Down
2 changes: 1 addition & 1 deletion go.work
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
go 1.22
go 1.22.3

use (
.
Expand Down
52 changes: 52 additions & 0 deletions testing/coreruleset/.ftw-overrides.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
version: "v0.0.0"
meta:
engine: "coraza"
platform: "go"
annotations:
- purpose: "Overrides for CRS test suite running against Coraza deployed as a Go middleware"
test_overrides:
# - rule_id: 920100
# test_ids: [5]
# reason: "Invalid uri, Coraza not reached - 301 returned"
# output:
# status: 301
- rule_id: 920100
test_ids: [8]
reason: |
On Apache is not allowed to put a colon in the path. Go/http allows it.
Note that the colon is a legal character in a regular path segment, according to the RFC.
output:
status: 200
log:
no_expect_ids: [920100]
- rule_id: 920270
test_ids: [4]
reason: "Rule works, Apache test expects status 400"
output:
log:
expect_ids: [920270]
- rule_id: 920274
test_ids: [1]
reason: "Host validation. Apache expects status 400. Coraza correctly triggers the rule 920274"
output:
log:
expect_ids: [920274]
- rule_id: 920290
test_ids: [1]
reason: "TODO"
output:
log:
expect_ids: [920280] # TODO: understand why 920280 (Missing Host Header) is triggered and not 920290 (Empty Host header). See what go-ftw sends.
- rule_id: 920290
test_ids: [4]
reason: "TODO"
output:
log:
expect_ids: [920280] # TODO: understand why 920280 (Missing Host Header) is triggered and not 920290 (Empty Host header). See what go-ftw sends.
- rule_id: 920430
test_ids: [8]
reason: "Go/http does not allow HTTP/3.0 - 505 HTTP Version Not Supported"
output:
status: 505
log:
no_expect_ids: [920430]
20 changes: 13 additions & 7 deletions testing/coreruleset/.ftw.yml
Original file line number Diff line number Diff line change
@@ -1,13 +1,19 @@
# Tests should not just be ignored via .ftw.yml, but new expectations for each test should be set.
# Avoid as much as possible adding new entries here, in favor of .ftw-overrides.yml
---
testoverride:
ignore:
920100-4: 'Invalid uri, Coraza not reached - 404 page not found'
920100-5: 'Invalid uri, Coraza not reached - 404 page not found'
920100-8: 'Go/http allows a colon in the path. Test expects status 400 or 403 (Apache behaviour)'
920270-4: 'Rule works, log contains 920270. Test expects status 400 (Apache behaviour)'
920272-5: 'Rule works, log contains 920272. Test expects status 400 (Apache behaviour)'
920290-1: 'Rule works, log contains 920290. Test expects status 400 (Apache behaviour)'
920290-4: 'Go/http returns 400 Bad Request: missing required Host header'
920430-8: 'Go/http does not allow HTTP/3.0 - 505 HTTP Version Not Supported'
932200-13: 'wip'
930110-7: 'CRS issue: https://github.com/coreruleset/coreruleset/issues/3736'

# TODO: investigate
932200-13: 'Failing only in multiphase evalution'
932300-10: 'Failing only in multiphase evalution'
933120-2: 'Failing only in multiphase evalution'
920280-3: ''
920430-3: ''
920430-5: ''
920430-9: ''
920610-2: 'fragments, Coraza might just happly accept them. Run and check it.'
920620-1: 'Rule checks if multiple Content-Type headers are kepts. Go/http might keep them and trigger the rule. Run and check it.'
57 changes: 17 additions & 40 deletions testing/coreruleset/coreruleset_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ package coreruleset

import (
"bufio"
b64 "encoding/base64"
"fmt"
"io"
"io/fs"
Expand All @@ -21,6 +20,7 @@ import (
"strconv"
"strings"
"testing"
"time"

"github.com/bmatcuk/doublestar/v4"
"github.com/coreruleset/go-ftw/config"
Expand All @@ -34,6 +34,7 @@ import (
"github.com/corazawaf/coraza/v3"
txhttp "github.com/corazawaf/coraza/v3/http"
"github.com/corazawaf/coraza/v3/types"
albedo "github.com/coreruleset/albedo/server"
)

func BenchmarkCRSCompilation(b *testing.B) {
Expand Down Expand Up @@ -220,43 +221,13 @@ SecRule REQUEST_HEADERS:X-CRS-Test "@rx ^.*$" \
t.Fatal(err)
}

// CRS regression tests are expected to be run with https://github.com/coreruleset/albedo as backend server
s := httptest.NewServer(txhttp.WrapHandler(waf, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
// TODO: Investigate why we need to enforce text/plain to have response body tests working.
// Check the Content-Type set by albed and SecResponseBodyMimeType
w.Header().Set("Content-Type", "text/plain")
switch {
case r.URL.Path == "/anything", r.URL.Path == "/post":
body, err := io.ReadAll(r.Body)
// Emulated httpbin behaviour: /anything and /post endpoints act as an echo server, writing back the request body
if r.Header.Get("Content-Type") == "application/x-www-form-urlencoded" {
// Tests 954120-1 and 954120-2 are the only two calling /anything with a POST and payload is urlencoded
if err != nil {
t.Fatalf("handler can not read request body: %v", err)
}
urldecodedBody, err := url.QueryUnescape(string(body))
if err != nil {
t.Logf("[warning] handler can not unescape urlencoded request body: %v", err)
// If the body can't be unescaped, we will keep going with the received body
urldecodedBody = string(body)
}
fmt.Fprint(w, urldecodedBody)
} else {
_, err = w.Write(body)
if err != nil {
t.Fatalf("handler can not write request body: %v", err)
}
}

case strings.HasPrefix(r.URL.Path, "/base64/"):
// Emulated httpbin behaviour: /base64 endpoint write the decoded base64 into the response body
b64Decoded, err := b64.StdEncoding.DecodeString(strings.TrimPrefix(r.URL.Path, "/base64/"))
if err != nil {
t.Fatalf("handler can not decode base64: %v", err)
}
fmt.Fprint(w, string(b64Decoded))
default:
// Common path "/status/200" defaults here
fmt.Fprint(w, "Hello!")
}
albedo.Handler().ServeHTTP(w, r)
})))
defer s.Close()

Expand All @@ -266,7 +237,7 @@ SecRule REQUEST_HEADERS:X-CRS-Test "@rx ^.*$" \
if err != nil {
return err
}
ftwt, err := test.GetTestFromYaml(yaml)
ftwt, err := test.GetTestFromYaml(yaml, path)
if err != nil {
return err
}
Expand All @@ -293,15 +264,21 @@ SecRule REQUEST_HEADERS:X-CRS-Test "@rx ^.*$" \
cfg.TestOverride.Overrides.DestAddr = &host
cfg.TestOverride.Overrides.Port = &port

cfg.LoadPlatformOverrides(".ftw-overrides.yml")
res, err := runner.Run(cfg, tests, runner.RunnerConfig{
ShowTime: false,
ShowTime: false,
ReadTimeout: 3 * time.Second, // Defaults to 1s but looks to be not enough in the CI
}, output.NewOutput("quiet", os.Stdout))
if err != nil {
t.Fatal(err)
}

if len(res.Stats.Failed) > 0 {
t.Errorf("failed tests: %v", res.Stats.Failed)
totalIgnored := len(res.Stats.Ignored)
if totalIgnored > 0 {
t.Logf("[info] %d ignored tests: %v", totalIgnored, res.Stats.Ignored)
}
totalFailed := len(res.Stats.Failed)
if totalFailed > 0 {
t.Errorf("[fatal] %d failed tests: %v", totalFailed, res.Stats.Failed)
}
}

Expand Down
26 changes: 15 additions & 11 deletions testing/coreruleset/go.mod
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
module github.com/corazawaf/coraza/v3/testing/coreruleset

go 1.22
go 1.22.3

require (
github.com/bmatcuk/doublestar/v4 v4.6.1
github.com/corazawaf/coraza-coreruleset/v4 v4.3.0
github.com/corazawaf/coraza-coreruleset/v4 v4.5.0
github.com/corazawaf/coraza/v3 v3.0.0-00010101000000-000000000000
github.com/coreruleset/go-ftw v0.6.4
github.com/coreruleset/albedo v0.0.16-0.20240924185852-4b95a321ebfd
github.com/coreruleset/go-ftw v1.0.4-0.20240809050408-f8169f0325ac
github.com/rs/zerolog v1.33.0
)

Expand All @@ -15,8 +16,8 @@ require (
github.com/Masterminds/semver v1.5.0 // indirect
github.com/Masterminds/sprig v2.22.0+incompatible // indirect
github.com/corazawaf/libinjection-go v0.2.1 // indirect
github.com/coreruleset/ftw-tests-schema v1.1.0 // indirect
github.com/fatih/color v1.16.0 // indirect
github.com/coreruleset/ftw-tests-schema/v2 v2.1.0 // indirect
github.com/fatih/color v1.17.0 // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 // indirect
github.com/goccy/go-yaml v1.11.3 // indirect
Expand All @@ -28,27 +29,30 @@ require (
github.com/knadh/koanf/maps v0.1.1 // indirect
github.com/knadh/koanf/parsers/yaml v0.1.0 // indirect
github.com/knadh/koanf/providers/env v0.1.0 // indirect
github.com/knadh/koanf/providers/file v0.1.0 // indirect
github.com/knadh/koanf/providers/file v1.1.0 // indirect
github.com/knadh/koanf/providers/rawbytes v0.1.0 // indirect
github.com/knadh/koanf/v2 v2.1.1 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/kyokomi/emoji/v2 v2.2.13 // indirect
github.com/magefile/mage v1.15.0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mitchellh/copystructure v1.2.0 // indirect
github.com/mitchellh/reflectwalk v1.0.2 // indirect
github.com/petar-dambovaliev/aho-corasick v0.0.0-20240411101913-e07a1f0e8eb4 // indirect
github.com/stretchr/testify v1.9.0 // indirect
github.com/tidwall/gjson v1.17.1 // indirect
github.com/rogpeppe/go-internal v1.13.1 // indirect
github.com/tidwall/gjson v1.17.3 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.1 // indirect
github.com/valllabh/ocsf-schema-golang v1.0.3 // indirect
github.com/yargevad/filepathx v1.0.0 // indirect
golang.org/x/crypto v0.26.0 // indirect
golang.org/x/net v0.28.0 // indirect
golang.org/x/crypto v0.27.0 // indirect
golang.org/x/net v0.29.0 // indirect
golang.org/x/sync v0.8.0 // indirect
golang.org/x/sys v0.23.0 // indirect
golang.org/x/sys v0.25.0 // indirect
golang.org/x/time v0.6.0 // indirect
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
google.golang.org/protobuf v1.34.1 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
rsc.io/binaryregexp v0.2.0 // indirect
)
Expand Down
Loading

0 comments on commit c531a97

Please sign in to comment.