-
Notifications
You must be signed in to change notification settings - Fork 30
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: add quantitative testing (#355)
* feat: add quantitative testing Signed-off-by: Felipe Zipitria <felipe.zipitria@owasp.org> * Apply suggestions from code review Co-authored-by: Max Leske <250711+theseion@users.noreply.github.com> * feat: move to interfaces Signed-off-by: Felipe Zipitria <felipe.zipitria@owasp.org> * test: add more coverage Signed-off-by: Felipe Zipitria <felipe.zipitria@owasp.org> * fix: apply code review suggestions Signed-off-by: Felipe Zipitria <felipe.zipitria@owasp.org> * docs: add basic documentation Signed-off-by: Felipe Zipitria <felipe.zipitria@owasp.org> * chore: cleanup comment Signed-off-by: Felipe Zipitria <felipe.zipitria@owasp.org> * fix: counting FPs Signed-off-by: Felipe Zipitria <felipe.zipitria@owasp.org> * fix: use both phases for getting FPs Signed-off-by: Felipe Zipitria <felipe.zipitria@owasp.org> * docs: add more examples Signed-off-by: Felipe Zipitria <felipe.zipitria@owasp.org> * fix: reduce noise in output Signed-off-by: Felipe Zipitria <felipe.zipitria@owasp.org> * feat: add factories for creating new objects Signed-off-by: Felipe Zipitria <felipe.zipitria@owasp.org> * test: simplify file tests Signed-off-by: Felipe Zipitria <felipe.zipitria@owasp.org> * fix: apply suggestions from code review Co-authored-by: Max Leske <250711+theseion@users.noreply.github.com> * fix: go.sum Signed-off-by: Felipe Zipitria <felipe.zipitria@owasp.org> --------- Signed-off-by: Felipe Zipitria <felipe.zipitria@owasp.org> Co-authored-by: Max Leske <250711+theseion@users.noreply.github.com>
- Loading branch information
Showing
22 changed files
with
2,643 additions
and
24 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,103 @@ | ||
| // Copyright 2023 OWASP ModSecurity Core Rule Set Project | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| package cmd | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "os" | ||
|
|
||
| "github.com/spf13/cobra" | ||
|
|
||
| "github.com/coreruleset/go-ftw/experimental/corpus" | ||
| "github.com/coreruleset/go-ftw/internal/quantitative" | ||
| "github.com/coreruleset/go-ftw/output" | ||
| ) | ||
|
|
||
| // NewQuantitativeCmd | ||
| // Returns a new cobra command for running quantitative tests | ||
| func NewQuantitativeCmd() *cobra.Command { | ||
| runCmd := &cobra.Command{ | ||
| Use: "quantitative", | ||
| Short: "Run quantitative tests", | ||
| Long: `Run all quantitative tests`, | ||
| RunE: runQuantitativeE, | ||
| } | ||
|
|
||
| runCmd.Flags().IntP("lines", "l", 0, "Number of lines of input to process before stopping") | ||
| runCmd.Flags().IntP("paranoia-level", "P", 1, "Paranoia level used to run the quantitative tests") | ||
| runCmd.Flags().IntP("corpus-line", "n", 0, "Number is the payload line from the corpus to exclusively send") | ||
| runCmd.Flags().StringP("payload", "p", "", "Payload is a string you want to test using quantitative tests. Will not use the corpus.") | ||
| runCmd.Flags().IntP("rule", "r", 0, "Rule ID of interest: only show false positives for specified rule ID") | ||
| runCmd.Flags().StringP("corpus", "c", "leipzig", "Corpus to use for the quantitative tests") | ||
| runCmd.Flags().StringP("corpus-lang", "L", "eng", "Corpus language to use for the quantitative tests") | ||
| runCmd.Flags().StringP("corpus-size", "s", "100K", "Corpus size to use for the quantitative tests. Most corpora will have sizes like \"100K\", \"1M\", etc.") | ||
| runCmd.Flags().StringP("corpus-year", "y", "2023", "Corpus year to use for the quantitative tests. Most corpus will have a year like \"2023\", \"2022\", etc.") | ||
| runCmd.Flags().StringP("corpus-source", "S", "news", "Corpus source to use for the quantitative tests. Most corpus will have a source like \"news\", \"web\", \"wikipedia\", etc.") | ||
| runCmd.Flags().StringP("directory", "d", ".", "Directory where the CRS rules are stored") | ||
| runCmd.Flags().StringP("file", "f", "", "Output file path for quantitative tests. Prints to standard output by default.") | ||
| runCmd.Flags().StringP("output", "o", "normal", "Output type for quantitative tests. \"normal\" is the default.") | ||
|
|
||
| return runCmd | ||
| } | ||
|
|
||
| func runQuantitativeE(cmd *cobra.Command, _ []string) error { | ||
| cmd.SilenceUsage = true | ||
|
|
||
| corpusTypeAsString, _ := cmd.Flags().GetString("corpus") | ||
| corpusSize, _ := cmd.Flags().GetString("corpus-size") | ||
| corpusLang, _ := cmd.Flags().GetString("corpus-lang") | ||
| corpusYear, _ := cmd.Flags().GetString("corpus-year") | ||
| corpusSource, _ := cmd.Flags().GetString("corpus-source") | ||
| directory, _ := cmd.Flags().GetString("directory") | ||
| fast, _ := cmd.Flags().GetInt("fast") | ||
| lines, _ := cmd.Flags().GetInt("lines") | ||
| outputFilename, _ := cmd.Flags().GetString("file") | ||
| paranoiaLevel, _ := cmd.Flags().GetInt("paranoia-level") | ||
| payload, _ := cmd.Flags().GetString("payload") | ||
| number, _ := cmd.Flags().GetInt("number") | ||
| rule, _ := cmd.Flags().GetInt("rule") | ||
| wantedOutput, _ := cmd.Flags().GetString("output") | ||
|
|
||
| if paranoiaLevel > 1 && rule > 0 { | ||
| return fmt.Errorf("paranoia level and rule ID cannot be used together") | ||
| } | ||
|
|
||
| // use outputFile to write to file | ||
| var outputFile *os.File | ||
| var err error | ||
| if outputFilename == "" { | ||
| outputFile = os.Stdout | ||
| } else { | ||
| outputFile, err = os.Open(outputFilename) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| } | ||
| out := output.NewOutput(wantedOutput, outputFile) | ||
|
|
||
| var corpusType corpus.Type | ||
| if corpusTypeAsString != "" { | ||
| err = corpusType.Set(corpusTypeAsString) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| } | ||
|
|
||
| params := quantitative.Params{ | ||
| Corpus: corpusType, | ||
| CorpusSize: corpusSize, | ||
| CorpusYear: corpusYear, | ||
| CorpusLang: corpusLang, | ||
| CorpusSource: corpusSource, | ||
| Directory: directory, | ||
| Fast: fast, | ||
| Lines: lines, | ||
| ParanoiaLevel: paranoiaLevel, | ||
| Number: number, | ||
| Payload: payload, | ||
| Rule: rule, | ||
| } | ||
|
|
||
| return quantitative.RunQuantitativeTests(params, out) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,62 @@ | ||
| // Copyright 2023 OWASP ModSecurity Core Rule Set Project | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| package cmd | ||
|
|
||
| import ( | ||
| "context" | ||
| "io/fs" | ||
| "os" | ||
| "path" | ||
| "testing" | ||
|
|
||
| "github.com/spf13/cobra" | ||
| "github.com/stretchr/testify/suite" | ||
| ) | ||
|
|
||
| var crsSetupFileContents = `# CRS Setup Configuration filename` | ||
| var emptyRulesFile = `# Empty Rules filename` | ||
|
|
||
| type quantitativeCmdTestSuite struct { | ||
| suite.Suite | ||
| tempDir string | ||
| rootCmd *cobra.Command | ||
| } | ||
|
|
||
| func TestQuantitativeTestSuite(t *testing.T) { | ||
| suite.Run(t, new(quantitativeCmdTestSuite)) | ||
| } | ||
|
|
||
| func (s *quantitativeCmdTestSuite) SetupTest() { | ||
| s.rootCmd = NewRootCommand() | ||
| s.tempDir = s.T().TempDir() | ||
|
|
||
| err := os.MkdirAll(path.Join(s.tempDir, "rules"), fs.ModePerm) | ||
| s.Require().NoError(err) | ||
| fakeCrsSetupConf, err := os.Create(path.Join(s.tempDir, "crs-setup.conf.example")) | ||
| s.Require().NoError(err) | ||
| n, err := fakeCrsSetupConf.WriteString(crsSetupFileContents) | ||
| s.Require().NoError(err) | ||
| s.Equal(len(crsSetupFileContents), n) | ||
| err = fakeCrsSetupConf.Close() | ||
| s.Require().NoError(err) | ||
| fakeRulesFile, err := os.Create(path.Join(s.tempDir, "rules", "Rules1.conf")) | ||
| s.Require().NoError(err) | ||
| n, err = fakeRulesFile.WriteString(emptyRulesFile) | ||
| s.Require().NoError(err) | ||
| s.Equal(len(emptyRulesFile), n) | ||
| s.rootCmd.AddCommand(NewQuantitativeCmd()) | ||
| } | ||
|
|
||
| func (s *quantitativeCmdTestSuite) TearDownTest() { | ||
| err := os.RemoveAll(s.tempDir) | ||
| s.Require().NoError(err) | ||
| } | ||
|
|
||
| func (s *quantitativeCmdTestSuite) TestQuantitativeCommand() { | ||
| s.rootCmd.SetArgs([]string{"quantitative", "-d", s.tempDir}) | ||
| cmd, err := s.rootCmd.ExecuteContextC(context.Background()) | ||
| s.Require().NoError(err, "quantitative command should not return error") | ||
| s.Equal("quantitative", cmd.Name(), "quantitative command should have the name 'quantitative'") | ||
| s.Require().NoError(err) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.