diff --git a/cmd/quantitative.go b/cmd/quantitative.go index 396551b..ea1d9f2 100644 --- a/cmd/quantitative.go +++ b/cmd/quantitative.go @@ -9,6 +9,7 @@ import ( "github.com/spf13/cobra" + "github.com/coreruleset/go-ftw/experimental/corpus" "github.com/coreruleset/go-ftw/internal/quantitative" "github.com/coreruleset/go-ftw/output" ) @@ -43,7 +44,7 @@ func NewQuantitativeCmd() *cobra.Command { func runQuantitativeE(cmd *cobra.Command, _ []string) error { cmd.SilenceUsage = true - corpus, _ := cmd.Flags().GetString("corpus") + corpusTypeAsString, _ := cmd.Flags().GetString("corpus") corpusSize, _ := cmd.Flags().GetString("corpus-size") corpusLang, _ := cmd.Flags().GetString("corpus-lang") corpusYear, _ := cmd.Flags().GetString("corpus-year") @@ -75,8 +76,16 @@ func runQuantitativeE(cmd *cobra.Command, _ []string) error { } out := output.NewOutput(wantedOutput, outputFile) - params := quantitative.QuantitativeParams{ - Corpus: corpus, + var corpusType corpus.Type + if corpusTypeAsString != "" { + err = corpusType.Set(corpusTypeAsString) + if err != nil { + return err + } + } + + params := quantitative.Params{ + Corpus: corpusType, CorpusSize: corpusSize, CorpusYear: corpusYear, CorpusLang: corpusLang, diff --git a/experimental/corpus/types.go b/experimental/corpus/types.go index e0ba21e..bc3e194 100644 --- a/experimental/corpus/types.go +++ b/experimental/corpus/types.go @@ -18,13 +18,29 @@ // interface is subject to change. package corpus -// Define an enum for CorpusType +import "fmt" + +// Type is the type of the corpus type Type string const ( Leipzig Type = "leipzig" ) +func (t *Type) String() string { + return string(*t) +} + +func (t *Type) Set(value string) error { + switch value { + case "leipzig": + *t = Leipzig + return nil + default: + return fmt.Errorf("invalid option for Type: '%s'", value) + } +} + // File interface is used to interact with Corpus files. // It provides methods for setting the cache directory and file path. type File interface { diff --git a/internal/quantitative/leipzig/corpus.go b/internal/quantitative/leipzig/corpus.go index a0e2c23..de57e5d 100644 --- a/internal/quantitative/leipzig/corpus.go +++ b/internal/quantitative/leipzig/corpus.go @@ -73,22 +73,24 @@ func NewLeipzigCorpus() corpus.Corpus { return leipzig } -// size returns the size of the corpus +// Size returns the size of the corpus func (c *LeipzigCorpus) Size() string { return c.size } +// WithSize sets the size of the corpus func (c *LeipzigCorpus) WithSize(size string) corpus.Corpus { c.size = size c.regenerateFileNames() return c } -// year returns the year of the corpus +// Year returns the year of the corpus func (c *LeipzigCorpus) Year() string { return c.year } +// WithYear sets the year of the corpus func (c *LeipzigCorpus) WithYear(year string) corpus.Corpus { c.year = year c.regenerateFileNames() @@ -112,17 +114,19 @@ func (c *LeipzigCorpus) Source() string { return c.source } +// WithSource sets the source of the corpus func (c *LeipzigCorpus) WithSource(source string) corpus.Corpus { c.source = source c.regenerateFileNames() return c } -// Lang returns the language of the corpus +// Language returns the language of the corpus func (c *LeipzigCorpus) Language() string { return c.lang } +// WithLanguage sets the language of the corpus func (c *LeipzigCorpus) WithLanguage(lang string) corpus.Corpus { c.lang = lang c.regenerateFileNames() @@ -165,7 +169,11 @@ func (c *LeipzigCorpus) FetchCorpusFile() corpus.File { log.Fatal().Err(err).Msg("Could not create destination directory") } - cache := NewFile().WithCacheDir(cacheDir) + cache := NewFile().WithCacheDir(cacheDir).WithFilePath(c.filename) + + if cache.FilePath() == "" { + log.Fatal().Msg("Cache file path is empty") + } if info, err := os.Stat(path.Join(home, ".ftw", cache.FilePath())); err == nil { log.Debug().Msgf("filename %s already exists", info.Name()) diff --git a/internal/quantitative/leipzig/corpus_test.go b/internal/quantitative/leipzig/corpus_test.go index a80f1f6..314d626 100644 --- a/internal/quantitative/leipzig/corpus_test.go +++ b/internal/quantitative/leipzig/corpus_test.go @@ -36,8 +36,28 @@ func (s *leipzigCorpusTestSuite) TestWithSize() { s.Require().Equal("300K", s.corpus.Size()) } +func (s *leipzigCorpusTestSuite) TestWithYear() { + s.corpus.WithYear("2024") + s.Require().Equal("2024", s.corpus.Year()) +} + +func (s *leipzigCorpusTestSuite) TestWithSource() { + s.corpus.WithSource("news") + s.Require().Equal("news", s.corpus.Source()) +} + +func (s *leipzigCorpusTestSuite) TestWithLanguage() { + s.corpus.WithLanguage("eng") + s.Require().Equal("eng", s.corpus.Language()) +} + +func (s *leipzigCorpusTestSuite) TestWithURL() { + s.corpus.WithURL("https://downloads.wortschatz-leipzig.de/corpora") + s.Require().Equal("https://downloads.wortschatz-leipzig.de/corpora", s.corpus.URL()) +} + func (s *leipzigCorpusTestSuite) TestGetIterator() { - s.corpus.WithSize("10K") + s.corpus = s.corpus.WithSize("10K") s.cache = s.corpus.FetchCorpusFile() s.iter = s.corpus.GetIterator(s.cache) } diff --git a/internal/quantitative/leipzig/file_test.go b/internal/quantitative/leipzig/file_test.go new file mode 100644 index 0000000..1fcb878 --- /dev/null +++ b/internal/quantitative/leipzig/file_test.go @@ -0,0 +1,186 @@ +// Copyright 2024 OWASP CRS Project +// SPDX-License-Identifier: Apache-2.0 + +package leipzig + +import ( + "reflect" + "testing" + + "github.com/coreruleset/go-ftw/experimental/corpus" +) + +func TestFile_CacheDir(t *testing.T) { + type fields struct { + cacheDir string + filePath string + } + tests := []struct { + name string + fields fields + want string + }{ + { + name: "Test 1", + fields: fields{ + cacheDir: "cacheDir", + filePath: "filePath", + }, + want: "cacheDir", + }, + { + name: "Test 2", + fields: fields{ + cacheDir: "cacheDir2", + filePath: "filePath2", + }, + want: "cacheDir2", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f := File{ + cacheDir: tt.fields.cacheDir, + filePath: tt.fields.filePath, + } + if got := f.CacheDir(); got != tt.want { + t.Errorf("CacheDir() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestFile_FilePath(t *testing.T) { + type fields struct { + cacheDir string + filePath string + } + tests := []struct { + name string + fields fields + want string + }{ + { + name: "Test 1", + fields: fields{ + cacheDir: "cacheDir", + filePath: "filePath", + }, + want: "filePath", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f := File{ + cacheDir: tt.fields.cacheDir, + filePath: tt.fields.filePath, + } + if got := f.FilePath(); got != tt.want { + t.Errorf("FilePath() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestFile_WithCacheDir(t *testing.T) { + type fields struct { + cacheDir string + filePath string + } + type args struct { + cacheDir string + } + tests := []struct { + name string + fields fields + args args + want corpus.File + }{ + { + name: "Test 1", + fields: fields{ + cacheDir: "cacheDir1", + filePath: "filePath", + }, + args: args{ + cacheDir: "cacheDir10", + }, + want: File{ + cacheDir: "cacheDir10", + filePath: "filePath", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f := File{ + cacheDir: tt.fields.cacheDir, + filePath: tt.fields.filePath, + } + if got := f.WithCacheDir(tt.args.cacheDir); !reflect.DeepEqual(got, tt.want) { + t.Errorf("WithCacheDir() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestFile_WithFilePath(t *testing.T) { + type fields struct { + cacheDir string + filePath string + } + type args struct { + filePath string + } + tests := []struct { + name string + fields fields + args args + want corpus.File + }{ + { + name: "Test 1", + fields: fields{ + cacheDir: "cacheDir", + filePath: "filePath1", + }, + args: args{ + filePath: "filePath2", + }, + want: File{ + cacheDir: "cacheDir", + filePath: "filePath2", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f := File{ + cacheDir: tt.fields.cacheDir, + filePath: tt.fields.filePath, + } + if got := f.WithFilePath(tt.args.filePath); !reflect.DeepEqual(got, tt.want) { + t.Errorf("WithFilePath() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestNewFile(t *testing.T) { + tests := []struct { + name string + want corpus.File + }{ + { + name: "Test 1", + want: File{}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := NewFile(); !reflect.DeepEqual(got, tt.want) { + t.Errorf("NewFile() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/quantitative/local_engine_test.go b/internal/quantitative/local_engine_test.go index 15af9e6..a347db9 100644 --- a/internal/quantitative/local_engine_test.go +++ b/internal/quantitative/local_engine_test.go @@ -4,6 +4,7 @@ package quantitative import ( + "fmt" "net/http" "os" "path" @@ -13,7 +14,10 @@ import ( "github.com/stretchr/testify/suite" ) -const crsUrl = "https://github.com/coreruleset/coreruleset/releases/download/v4.6.0/coreruleset-4.6.0-minimal.tar.gz" +const ( + crsUrl = "https://github.com/coreruleset/coreruleset/releases/download/v4.6.0/coreruleset-4.6.0-minimal.tar.gz" + crsTestVersion = "4.6.0" +) type localEngineTestSuite struct { suite.Suite @@ -36,7 +40,7 @@ func (s *localEngineTestSuite) SetupTest() { err := client.Get() s.Require().NoError(err) - s.engine = NewEngine(path.Join(s.dir, "coreruleset-4.6.0"), 1) + s.engine = NewEngine(path.Join(s.dir, fmt.Sprintf("coreruleset-%s", crsTestVersion)), 1) s.Require().NotNil(s.engine) } diff --git a/internal/quantitative/runner.go b/internal/quantitative/runner.go index 345be6c..adc97eb 100644 --- a/internal/quantitative/runner.go +++ b/internal/quantitative/runner.go @@ -14,8 +14,8 @@ import ( "github.com/coreruleset/go-ftw/output" ) -// QuantitativeParams holds the parameters for the quantitative tests -type QuantitativeParams struct { +// Params holds the parameters for the quantitative tests +type Params struct { // Lines is the number of lines of input to process before stopping Lines int // Fast is the process 1 in every X lines of input ('fast run' mode) @@ -33,7 +33,7 @@ type QuantitativeParams struct { // CorpusSize is the corpus size to use for the quantitative tests CorpusSize string // Corpus is the corpus to use for the quantitative tests - Corpus string + Corpus corpus.Type // CorpusLang is the language of the corpus CorpusLang string // CorpusYear is the year of the corpus @@ -54,7 +54,7 @@ func NewCorpus(corpusType corpus.Type) corpus.Corpus { } // RunQuantitativeTests runs all quantitative tests -func RunQuantitativeTests(params QuantitativeParams, out *output.Output) error { +func RunQuantitativeTests(params Params, out *output.Output) error { out.Println("Running quantitative tests") log.Trace().Msgf("Lines: %d", params.Lines) diff --git a/internal/quantitative/runner_test.go b/internal/quantitative/runner_test.go new file mode 100644 index 0000000..275f7cb --- /dev/null +++ b/internal/quantitative/runner_test.go @@ -0,0 +1,74 @@ +// Copyright 2024 OWASP CRS Project +// SPDX-License-Identifier: Apache-2.0 + +package quantitative + +import ( + "bytes" + "fmt" + "os" + "path" + "testing" + + "github.com/hashicorp/go-getter" + "github.com/stretchr/testify/suite" + + "github.com/coreruleset/go-ftw/experimental/corpus" + "github.com/coreruleset/go-ftw/output" +) + +type runnerTestSuite struct { + suite.Suite + params Params + c corpus.Corpus + dir string +} + +func TestRunnerTestSuite(t *testing.T) { + suite.Run(t, new(runnerTestSuite)) +} + +func (s *runnerTestSuite) SetupTest() { + s.params = Params{ + Lines: 1000, + Fast: 10, + Rule: 1000, + Payload: "test", + Number: 1000, + Directory: path.Join(s.dir, fmt.Sprintf("coreruleset-%s", crsTestVersion)), + ParanoiaLevel: 1, + CorpusSize: "10K", + Corpus: "leipzig", + CorpusLang: "eng", + CorpusYear: "2023", + CorpusSource: "news", + } + s.dir = path.Join(os.TempDir()) + s.Require().NoError(os.MkdirAll(s.dir, 0755)) + client := &getter.Client{ + Mode: getter.ClientModeAny, + Src: crsUrl, + Dst: s.dir, + } + + err := client.Get() + s.Require().NoError(err) +} + +func (s *runnerTestSuite) TeardownTest() { + err := os.RemoveAll(s.dir) + s.Require().NoError(err) +} + +func (s *runnerTestSuite) TestNewCorpus() { + s.c = NewCorpus(corpus.Leipzig) + s.Require().NotNil(s.c) + s.Require().Equal(s.c.URL(), "https://downloads.wortschatz-leipzig.de/corpora") +} + +func (s *runnerTestSuite) TestRunQuantitativeTests() { + var b bytes.Buffer + out := output.NewOutput("plain", &b) + err := RunQuantitativeTests(s.params, out) + s.Require().NoError(err) +} diff --git a/internal/quantitative/stats.go b/internal/quantitative/stats.go index 6969598..7fa63a3 100644 --- a/internal/quantitative/stats.go +++ b/internal/quantitative/stats.go @@ -58,6 +58,11 @@ func (s *QuantitativeRunStats) addFalsePositive(rule int) { s.falsePositivesPerRule[rule]++ } +// FalsePositives returns the total false positives detected +func (s *QuantitativeRunStats) FalsePositives() int { + return s.falsePositives +} + // incrementRun increments the amount of tests executed in this run. func (s *QuantitativeRunStats) incrementRun() { s.count_++ diff --git a/internal/quantitative/stats_test.go b/internal/quantitative/stats_test.go new file mode 100644 index 0000000..b615ebb --- /dev/null +++ b/internal/quantitative/stats_test.go @@ -0,0 +1,131 @@ +// Copyright 2024 OWASP CRS Project +// SPDX-License-Identifier: Apache-2.0 + +package quantitative + +import ( + "bytes" + "testing" + "time" + + "github.com/stretchr/testify/suite" + + "github.com/coreruleset/go-ftw/output" +) + +type statsTestSuite struct { + suite.Suite +} + +func TestStatsTestSuite(t *testing.T) { + suite.Run(t, new(statsTestSuite)) +} + +func (s *statsTestSuite) TestNewQuantitativeStats() { + tests := []struct { + name string + want *QuantitativeRunStats + }{ + { + name: "Test 1", + want: &QuantitativeRunStats{ + count_: 0, + falsePositives: 0, + falsePositivesPerRule: make(map[int]int), + totalTime: 0, + }, + }, + } + for _, tt := range tests { + s.Run(tt.name, func() { + got := NewQuantitativeStats() + s.Require().Equal(got, tt.want) + }) + } +} + +func (s *statsTestSuite) TestQuantitativeRunStats_MarshalJSON() { + type fields struct { + count_ int + totalTime time.Duration + falsePositives int + falsePositivesPerRule map[int]int + } + tests := []struct { + name string + fields fields + want []byte + wantErr bool + }{ + { + name: "Test 1", + fields: fields{ + count_: 1, + totalTime: 1, + falsePositives: 1, + falsePositivesPerRule: map[int]int{920010: 1}, + }, + want: []byte(`{"count":1,"falsePositives":1,"falsePositivesPerRule":{"920010":1},"totalTime":1}`), + wantErr: false, + }, + { + name: "Test 2", + fields: fields{ + count_: 2, + totalTime: 2, + falsePositives: 2, + falsePositivesPerRule: map[int]int{933100: 2}, + }, + want: []byte(`{"count":2,"falsePositives":2,"falsePositivesPerRule":{"933100":2},"totalTime":2}`), + wantErr: false, + }, + } + for _, tt := range tests { + s.Run(tt.name, func() { + q := &QuantitativeRunStats{ + count_: tt.fields.count_, + totalTime: tt.fields.totalTime, + falsePositives: tt.fields.falsePositives, + falsePositivesPerRule: tt.fields.falsePositivesPerRule, + } + got, err := q.MarshalJSON() + s.Require().NoError(err) + s.Require().Equal(got, tt.want) + }) + } +} + +func (s *statsTestSuite) TestQuantitativeRunStats_functions() { + q := NewQuantitativeStats() + + q.incrementRun() + s.Require().Equal(q.Count(), 1) + + q.addFalsePositive(920100) + s.Require().Equal(q.FalsePositives(), 1) + + q.incrementRun() + s.Require().Equal(q.Count(), 2) + + q.addFalsePositive(920200) + s.Require().Equal(q.FalsePositives(), 2) + + duration := time.Duration(5000) + q.SetTotalTime(duration) + s.Require().Equal(q.TotalTime(), duration) +} + +func (s *statsTestSuite) TestQuantitativeRunStats_printSummary() { + var b bytes.Buffer + out := output.NewOutput("plain", &b) + q := NewQuantitativeStats() + + q.incrementRun() + s.Require().Equal(q.Count(), 1) + + q.addFalsePositive(920100) + s.Require().Equal(q.FalsePositives(), 1) + + q.printSummary(out) + s.Require().Equal(b.String(), "Run 1 payloads in 0s\nTotal False positive ratio: 1/1 = 1.0000\nFalse positives per rule: map[920100:1]\n") +}