From f19698de71cbebce424475b5619718800eb6c6b0 Mon Sep 17 00:00:00 2001 From: Hubert Siwik Date: Wed, 19 Jun 2024 20:55:08 +0200 Subject: [PATCH] feat: make inclusion to the test case list based on matching tags (#317) Use tags to narrow down test cases to those labeled with strings matching the regex passed as a config parameter. --- README.md | 1 + cmd/run.go | 17 +++++++++++++-- config/config.go | 1 + config/config_test.go | 2 ++ config/types.go | 2 ++ runner/run.go | 25 +++++++++++++++------- runner/run_test.go | 18 ++++++++++++++++ runner/testdata/TestRunTests_Run.yaml | 6 ++++++ runner/types.go | 3 +++ utils/slice.go | 18 ++++++++++++++++ utils/slice_test.go | 30 +++++++++++++++++++++++++++ 11 files changed, 114 insertions(+), 9 deletions(-) create mode 100644 utils/slice.go create mode 100644 utils/slice_test.go diff --git a/README.md b/README.md index 99025284..5f970515 100644 --- a/README.md +++ b/README.md @@ -133,6 +133,7 @@ Flags: -r, --rate-limit duration Limit the request rate to the server to 1 request per specified duration. 0 is the default, and disables rate limiting. --read-timeout duration timeout for receiving responses during test execution (default 10s) --show-failures-only shows only the results of failed tests + -T, --include-tags string include tests tagged with labels matching this Go regular expression (e.g. to include all tests being tagged with "cookie", use "^cookie$"). -t, --time show time spent per test --wait-delay duration Time to wait between retries for all wait operations. (default 1s) --wait-for-connection-timeout duration Http connection timeout, The timeout includes connection time, any redirects, and reading the response body. (default 3s) diff --git a/cmd/run.go b/cmd/run.go index 0c4551ed..6fe86a4e 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -33,6 +33,7 @@ func NewRunCommand() *cobra.Command { runCmd.Flags().StringP("exclude", "e", "", "exclude tests matching this Go regular expression (e.g. to exclude all tests beginning with \"91\", use \"^91.*\"). \nIf you want more permanent exclusion, check the 'exclude' option in the config file.") runCmd.Flags().StringP("include", "i", "", "include only tests matching this Go regular expression (e.g. to include only tests beginning with \"91\", use \"^91.*\"). \\nIf you want more permanent inclusion, check the 'include' option in the config file.\"") + runCmd.Flags().StringP("include-tags", "T", "", "include tests tagged with labels matching this Go regular expression (e.g. to include all tests being tagged with \"cookie\", use \"^cookie$\").") runCmd.Flags().StringP("dir", "d", ".", "recursively find yaml tests in this directory") runCmd.Flags().StringP("output", "o", "normal", "output type for ftw tests. \"normal\" is the default.") runCmd.Flags().StringP("file", "f", "", "output file path for ftw tests. Prints to standard output by default.") @@ -65,6 +66,7 @@ func runE(cmd *cobra.Command, _ []string) error { cmd.SilenceUsage = true exclude, _ := cmd.Flags().GetString("exclude") include, _ := cmd.Flags().GetString("include") + includeTags, _ := cmd.Flags().GetString("include-tags") dir, _ := cmd.Flags().GetString("dir") outputFilename, _ := cmd.Flags().GetString("file") logFilePath, _ := cmd.Flags().GetString("log-file") @@ -113,11 +115,21 @@ func runE(cmd *cobra.Command, _ []string) error { var includeRE *regexp.Regexp if include != "" { - includeRE = regexp.MustCompile(include) + if includeRE, err = regexp.Compile(include); err != nil { + return fmt.Errorf("invalid --include regular expression: %w", err) + } } var excludeRE *regexp.Regexp if exclude != "" { - excludeRE = regexp.MustCompile(exclude) + if excludeRE, err = regexp.Compile(exclude); err != nil { + return fmt.Errorf("invalid --exclude regular expression: %w", err) + } + } + var includeTagsRE *regexp.Regexp + if includeTags != "" { + if includeTagsRE, err = regexp.Compile(includeTags); err != nil { + return fmt.Errorf("invalid --include-tags regular expression: %w", err) + } } // Add wait4x checkers @@ -167,6 +179,7 @@ func runE(cmd *cobra.Command, _ []string) error { currentRun, err := runner.Run(cfg, tests, runner.RunnerConfig{ Include: includeRE, Exclude: excludeRE, + IncludeTags: includeTagsRE, ShowTime: showTime, ShowOnlyFailed: showOnlyFailed, ConnectTimeout: connectTimeout, diff --git a/config/config.go b/config/config.go index d1734042..f6d02a00 100644 --- a/config/config.go +++ b/config/config.go @@ -28,6 +28,7 @@ func NewDefaultConfig() *FTWConfiguration { MaxMarkerLogLines: DefaultMaxMarkerLogLines, IncludeTests: make(map[*FTWRegexp]string), ExcludeTests: make(map[*FTWRegexp]string), + IncludeTags: make(map[*FTWRegexp]string), } return cfg } diff --git a/config/config_test.go b/config/config_test.go index 30900a0e..93b360b3 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -20,6 +20,8 @@ include: '^9.*': 'Include all tests starting with 9' exclude: '^920400-2$': 'Exclude this test' +include_tags: + '^cookie$': 'Run test tagged with this label' testoverride: input: dest_addr: 'httpbingo.org' diff --git a/config/types.go b/config/types.go index f9216b11..94809215 100644 --- a/config/types.go +++ b/config/types.go @@ -48,6 +48,8 @@ type FTWConfiguration struct { IncludeTests map[*FTWRegexp]string `koanf:"include"` // ExcludeTests is a list of tests to exclude (same as --exclude) ExcludeTests map[*FTWRegexp]string `koanf:"exclude"` + // IncludeTags is a list of tags matching tests to run (same as --tag) + IncludeTags map[*FTWRegexp]string `koanf:"include_tags"` } type PlatformOverrides struct { diff --git a/runner/run.go b/runner/run.go index d18f23a1..b8aa6a8c 100644 --- a/runner/run.go +++ b/runner/run.go @@ -6,7 +6,6 @@ package runner import ( "errors" "fmt" - "regexp" "time" schema "github.com/coreruleset/ftw-tests-schema/v2/types" @@ -52,6 +51,7 @@ func Run(cfg *config.FTWConfiguration, tests []*test.FTWTest, c RunnerConfig, ou RunnerConfig: &c, Include: c.Include, Exclude: c.Exclude, + IncludeTags: c.IncludeTags, ShowTime: c.ShowTime, Output: out, ShowOnlyFailed: c.ShowOnlyFailed, @@ -84,7 +84,7 @@ func RunTest(runContext *TestRunContext, ftwTest *test.FTWTest) error { for _, testCase := range ftwTest.Tests { // if we received a particular test ID, skip until we find it - if needToSkipTest(runContext.Include, runContext.Exclude, &testCase) { + if needToSkipTest(runContext, &testCase) { runContext.Stats.addResultToStats(Skipped, &testCase) continue } @@ -257,7 +257,11 @@ func markAndFlush(runContext *TestRunContext, dest *ftwhttp.Destination, stageID return nil, fmt.Errorf("can't find log marker. Am I reading the correct log? Log file: %s", runContext.Config.LogFile) } -func needToSkipTest(include *regexp.Regexp, exclude *regexp.Regexp, testCase *schema.Test) bool { +func needToSkipTest(runContext *TestRunContext, testCase *schema.Test) bool { + include := runContext.Include + exclude := runContext.Exclude + includeTags := runContext.IncludeTags + // never skip enabled explicit inclusions if include != nil { if include.MatchString(testCase.IdString()) { @@ -266,12 +270,19 @@ func needToSkipTest(include *regexp.Regexp, exclude *regexp.Regexp, testCase *sc } } - result := false + // if the test's tags do not match the passed ones + // it needs to be skipped + if includeTags != nil { + if !utils.MatchSlice(includeTags, testCase.Tags) { + return true + } + } + // if we need to exclude tests, and the ID matches, // it needs to be skipped if exclude != nil { if exclude.MatchString(testCase.IdString()) { - result = true + return true } } @@ -279,11 +290,11 @@ func needToSkipTest(include *regexp.Regexp, exclude *regexp.Regexp, testCase *sc // it needs to be skipped if include != nil { if !include.MatchString(testCase.IdString()) { - result = true + return true } } - return result + return false } func checkTestSanity(stage *schema.Stage) error { diff --git a/runner/run_test.go b/runner/run_test.go index c5ad413f..8db508b6 100644 --- a/runner/run_test.go +++ b/runner/run_test.go @@ -313,6 +313,24 @@ func (s *runTestSuite) TestRunTests_Run() { s.Equal(res.Stats.TotalFailed(), 0, "failed to exclude test") }) + s.Run("count tests tagged with `tag-10`", func() { + res, err := Run(s.cfg, s.ftwTests, RunnerConfig{ + IncludeTags: regexp.MustCompile("^tag-10$"), + }, s.out) + s.Require().NoError(err) + s.Len(res.Stats.Success, 1, "failed to incorporate tagged test") + s.Equal(res.Stats.TotalFailed(), 0, "failed to incorporate tagged test") + }) + + s.Run("count tests tagged with `tag-8` and `tag-10`", func() { + res, err := Run(s.cfg, s.ftwTests, RunnerConfig{ + IncludeTags: regexp.MustCompile("^tag-8$|^tag-10$"), + }, s.out) + s.Require().NoError(err) + s.Len(res.Stats.Success, 2, "failed to incorporate tagged test") + s.Equal(res.Stats.TotalFailed(), 0, "failed to incorporate tagged test") + }) + s.Run("test exceptions 1", func() { res, err := Run(s.cfg, s.ftwTests, RunnerConfig{ Include: regexp.MustCompile("-1.*"), diff --git a/runner/testdata/TestRunTests_Run.yaml b/runner/testdata/TestRunTests_Run.yaml index f182d47a..6ecb3a2f 100644 --- a/runner/testdata/TestRunTests_Run.yaml +++ b/runner/testdata/TestRunTests_Run.yaml @@ -17,6 +17,9 @@ tests: expect_error: False status: 200 - test_id: 8 + tags: + - tag-8 + - local description: "this test is number 8" stages: - input: @@ -29,6 +32,9 @@ tests: output: status: 200 - test_id: 10 + tags: + - tag-10 + - other stages: - input: dest_addr: "{{ .TestAddr }}" diff --git a/runner/types.go b/runner/types.go index bbc0d02c..d7ebae06 100644 --- a/runner/types.go +++ b/runner/types.go @@ -19,6 +19,8 @@ type RunnerConfig struct { Include *regexp.Regexp // Exclude is a regular expression to filter tests to exclude. If nil, no tests are excluded. Exclude *regexp.Regexp + // IncludeTags is a regular expression to filter tests to count the ones tagged with the mathing label. If nil, no impact on test runner. + IncludeTags *regexp.Regexp // ShowTime determines whether to show the time taken to run each test. ShowTime bool // ShowOnlyFailed will only output information related to failed tests @@ -43,6 +45,7 @@ type TestRunContext struct { RunnerConfig *RunnerConfig Include *regexp.Regexp Exclude *regexp.Regexp + IncludeTags *regexp.Regexp ShowTime bool ShowOnlyFailed bool Output *output.Output diff --git a/utils/slice.go b/utils/slice.go new file mode 100644 index 00000000..4b833c8b --- /dev/null +++ b/utils/slice.go @@ -0,0 +1,18 @@ +// Copyright 2023 OWASP ModSecurity Core Rule Set Project +// SPDX-License-Identifier: Apache-2.0 + +package utils + +import ( + "regexp" +) + +func MatchSlice(regex *regexp.Regexp, hayStack []string) bool { + for _, str := range hayStack { + if regex.MatchString(str) { + return true + } + } + + return false +} diff --git a/utils/slice_test.go b/utils/slice_test.go new file mode 100644 index 00000000..03e15c89 --- /dev/null +++ b/utils/slice_test.go @@ -0,0 +1,30 @@ +// Copyright 2023 OWASP ModSecurity Core Rule Set Project +// SPDX-License-Identifier: Apache-2.0 + +package utils + +import ( + "regexp" + "testing" + + "github.com/stretchr/testify/suite" +) + +type sliceTestSuite struct { + suite.Suite +} + +func TestSliceTestSuite(t *testing.T) { + suite.Run(t, new(timeTestSuite)) +} + +func (s *timeTestSuite) TestMatchSlice() { + re := regexp.MustCompile("^cookie$") + + s.False(MatchSlice(re, []string{})) + s.False(MatchSlice(re, []string{""})) + s.False(MatchSlice(re, []string{"cooke", "php"})) + s.False(MatchSlice(re, []string{"cookies", "php"})) + s.True(MatchSlice(re, []string{"cookie", "php"})) + s.True(MatchSlice(re, []string{"js", "cookie"})) +}