From b822d9c800a1c4c5e6534023a57ba3492e1b40ea Mon Sep 17 00:00:00 2001 From: Anton Korpusenko Date: Thu, 11 Apr 2024 16:12:11 +0300 Subject: [PATCH] fixed degradation test script --- scripts/degradation-tester/config.yaml | 5 +- .../degradation-tester/degradation-check.sh | 2 +- scripts/degradation-tester/go.mod | 2 +- scripts/degradation-tester/go.sum | 2 + scripts/degradation-tester/main.go | 194 +++++++----- scripts/degradation-tester/main_test.go | 294 +++++++++++++++--- scripts/degradation-tester/types.go | 80 +++++ .../degradation-tester/update-benchmarks.sh | 2 +- 8 files changed, 464 insertions(+), 117 deletions(-) create mode 100644 scripts/degradation-tester/types.go diff --git a/scripts/degradation-tester/config.yaml b/scripts/degradation-tester/config.yaml index b9583bf044..10110c2927 100644 --- a/scripts/degradation-tester/config.yaml +++ b/scripts/degradation-tester/config.yaml @@ -1,5 +1,6 @@ -DefaultOpDelta: 5.0 -DefaultAllocDelta: 5.0 +DefaultSecOpDelta: 5.0 +DefaultBOpDelta: 5.0 +DefaultAllocOpDelta: 5.0 Packages: - Path: "./message/validation" Tests: diff --git a/scripts/degradation-tester/degradation-check.sh b/scripts/degradation-tester/degradation-check.sh index 417d8ce7bf..f24f664b4e 100644 --- a/scripts/degradation-tester/degradation-check.sh +++ b/scripts/degradation-tester/degradation-check.sh @@ -17,7 +17,7 @@ for pkgPath in "${packagePaths[@]}"; do # count should be at least 10. Ideally 20 - go test -bench=. -count=10 -benchmem "$pkgPath" | tee "$outputFile" + go test -bench=. -count=16 -benchmem "$pkgPath" | tee "$outputFile" benchstat -format csv "$oldBenchmarks" "$outputFile" | tee "${benchStatFile}" diff --git a/scripts/degradation-tester/go.mod b/scripts/degradation-tester/go.mod index 73cb54d5d4..a6df6f02e1 100644 --- a/scripts/degradation-tester/go.mod +++ b/scripts/degradation-tester/go.mod @@ -6,7 +6,7 @@ require gopkg.in/yaml.v3 v3.0.1 require ( github.com/aclements/go-moremath v0.0.0-20210112150236-f10218a38794 // indirect - golang.org/x/perf v0.0.0-20240305160248-5eefbfdba9dd // indirect + golang.org/x/perf v0.0.0-20240404204407-f3e401e020e4 // indirect ) require ( diff --git a/scripts/degradation-tester/go.sum b/scripts/degradation-tester/go.sum index 9090f60ef6..c525041f18 100644 --- a/scripts/degradation-tester/go.sum +++ b/scripts/degradation-tester/go.sum @@ -8,6 +8,8 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= golang.org/x/perf v0.0.0-20240305160248-5eefbfdba9dd h1:rglj7j7GZzz4GcR21t9lmupN+8ALC8L//rdzE/50vLE= golang.org/x/perf v0.0.0-20240305160248-5eefbfdba9dd/go.mod h1:9aZNLn0je8D5R0rbpRog/X1gTnJt4uajOXR4k1WpzXk= +golang.org/x/perf v0.0.0-20240404204407-f3e401e020e4 h1:a+TLAEdWcQdugcYroBtJI8lJOTENK6my3T1ew+QGGu0= +golang.org/x/perf v0.0.0-20240404204407-f3e401e020e4/go.mod h1:us0Iv7UioeaOxNf4AhKdAwwTqEVfOUfzy2Z0Bu+beE0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/scripts/degradation-tester/main.go b/scripts/degradation-tester/main.go index 5673c01fa3..ea96f28636 100644 --- a/scripts/degradation-tester/main.go +++ b/scripts/degradation-tester/main.go @@ -4,7 +4,6 @@ import ( "bufio" "encoding/csv" "fmt" - "math" "os" "regexp" "strconv" @@ -16,23 +15,6 @@ import ( var opThresholds map[string]float64 var allocThresholds map[string]float64 -type Config struct { - DefaultOpDelta float64 `yaml:"DefaultOpDelta"` - DefaultAllocDelta float64 `yaml:"DefaultAllocDelta"` - Packages []BenchmarkingTestConfig `yaml:"Packages"` -} - -type BenchmarkingTestConfig struct { - Path string `yaml:"Path"` - Tests []TestCase `yaml:"Tests"` -} - -type TestCase struct { - Name string `yaml:"Name"` - OpDelta float64 `yaml:"OpDelta"` - AllocDelta float64 `yaml:"AllocDelta"` -} - func main() { if len(os.Args) < 3 { fmt.Println("Usage: degradation-tester ") @@ -42,7 +24,7 @@ func main() { configFilename := os.Args[1] resultsFilename := os.Args[2] - config, err := loadConfig(configFilename) + cfg, err := loadConfig(configFilename) if err != nil { fmt.Printf("Error loading conifg: %v", err) os.Exit(1) @@ -53,24 +35,59 @@ func main() { fmt.Printf("Error opening results file: %v", err) os.Exit(1) } - defer file.Close() + defer func() { + _ = file.Close() + }() - totalErrors := checkFile(file, config) + checkResult := checkFile(file, cfg) - if totalErrors > 0 { + if checkResult.TotalIssues > 0 { + for _, issue := range checkResult.Issues[SectionTypeBop] { + fmt.Println(issue) + } os.Exit(1) } } -func checkFile(file *os.File, config *Config) int { +func checkFile(file *os.File, cfg *Config) *DegradationCheckResult { var currentSection string + var currentPkgPath string + + var oldRes, newRes, curRes *BenchmarkResult scanner := bufio.NewScanner(file) - totalErrors := 0 + + ignoreBErrorsRegexp := regexp.MustCompile(`B\d+:`) for scanner.Scan() { line := scanner.Text() switch { + case line == "", + ignoreBErrorsRegexp.MatchString(line), + strings.HasPrefix(line, "pkg:"), + strings.HasPrefix(line, "cpu:"), + strings.HasPrefix(line, "goarch:"), + strings.HasPrefix(line, "goos:"), + strings.Contains(line, "geomean"): + continue + case strings.Contains(line, "old.txt"): + if oldRes != nil { + continue + } + row := MustParseCsvLine(line) + currentPkgPath = row[1] + oldRes = NewBenchmarkResult(currentPkgPath) + curRes = oldRes + continue + case strings.Contains(line, "new.txt"): + if newRes != nil { + continue + } + row := MustParseCsvLine(line) + currentPkgPath = row[1] + newRes = NewBenchmarkResult(currentPkgPath) + curRes = newRes + continue case strings.Contains(line, "sec/op"): currentSection = "sec/op" continue @@ -81,71 +98,46 @@ func checkFile(file *os.File, config *Config) int { currentSection = "allocs/op" continue } - if currentSection == "B/op" { - continue + + err := handleResultRow(line, currentSection, curRes) + if err != nil { + panic(err) } - totalErrors += checkLine(config, line, currentSection) } if err := scanner.Err(); err != nil { fmt.Printf("Error reading results file: %v\n", err) } - return totalErrors + return CheckDegradation(cfg, oldRes, newRes) } -func checkLine( - config *Config, +func handleResultRow( line string, section string, -) int { - if line == "" { - return 0 - } - csvRowReader := csv.NewReader(strings.NewReader(line)) - csvRowReader.Comment = '#' - csvRowReader.Comma = ',' - - row, err := csvRowReader.Read() - if err != nil { - fmt.Printf("failed parsing CSV line %s with erorr: %v\n", line, err) - os.Exit(1) - } + results *BenchmarkResult, +) error { + row := MustParseCsvLine(line) - if len(row) != 7 { - // ignore all except the becnhmark result lines with exactly 7 columns - return 0 - } - - // The "geomean" represents a statistical summary (geometric mean) of multiple test results, - // not an individual test result, hence we should just skip it - if row[0] == "geomean" { - return 0 + if len(row) != 3 { + return fmt.Errorf("invalid row: %v", row) } normalizedTestName := normalizeTestName(row[0]) - oldChangeStr := row[2] - oldChange, err := strconv.ParseFloat(strings.TrimSuffix(oldChangeStr, "%"), 64) - if err != nil { - fmt.Printf("⚠️ Error parsing float: %v\n", err) - return 1 - } + valueStr := row[1] + value, err := strconv.ParseFloat(strings.TrimSuffix(valueStr, "%"), 64) - newChangeStr := row[4] - newChange, err := strconv.ParseFloat(strings.TrimSuffix(newChangeStr, "%"), 64) if err != nil { - fmt.Printf("⚠️ Error parsing float: %v\n", err) - return 1 + return fmt.Errorf("⚠️ Error parsing value %s: %v\n", valueStr, err) } - threshold := getThresholdForTestCase(config, normalizedTestName, section) - - if math.Abs(oldChange-newChange) > threshold { - fmt.Printf("❌ Change in section %s for test %s exceeds threshold: %s\n", section, normalizedTestName, newChangeStr) - return 1 + results.Res[section][normalizedTestName] = &TestResult{ + Name: normalizedTestName, + Value: value, + CI: row[2], } - return 0 + return nil } func loadConfig(filename string) (*Config, error) { @@ -190,13 +182,75 @@ func getThresholdForTestCase( if threshold, exists := opThresholds[testName]; exists { return threshold } - return config.DefaultOpDelta + return config.DefaultSecOpDelta + case "B/op": + if threshold, exists := opThresholds[testName]; exists { + return threshold + } + return config.DefaultBOpDelta case "allocs/op": if threshold, exists := allocThresholds[testName]; exists { return threshold } - return config.DefaultAllocDelta + return config.DefaultAllocOpDelta default: return 0.0 } } + +func CheckDegradation( + cfg *Config, + oldRes *BenchmarkResult, + newRes *BenchmarkResult, +) *DegradationCheckResult { + checkRes := NewDegradationCheckResult(oldRes.Pkg) + for section, oldTests := range oldRes.Res { + newTests, exists := newRes.Res[section] + if !exists { + panic(fmt.Sprintf("❌ Section %s not found in new benchmarks of pkg %s. Please manualy update the benchmarks!\n", section, newRes.Pkg)) + } + + for testName, oldTest := range oldTests { + newTest, exists := newTests[testName] + if !exists { + panic(fmt.Sprintf("❌ Test %s not found in new benchmarks of pkg %s. Please manualy update the benchmarks!\n", testName, newRes.Pkg)) + } + + threshold := getThresholdForTestCase(cfg, testName, section) + + a := newTest.Value + b := oldTest.Value + + // Floats comparison. Swap values if `a` is greater than `b`. + if a-b > 1e-6 { + a, b = b, a + } + + // Calculate the absolute change in percentage + absChange := (b - a) / a * 100.0 + + if absChange > threshold { + checkRes.TotalIssues++ + checkRes.Issues[section] = append(checkRes.Issues[section], &DegradationIssue{ + TestName: testName, + Diff: absChange, + Threshold: threshold, + }) + } + } + } + return checkRes +} + +func MustParseCsvLine(line string) []string { + csvRowReader := csv.NewReader(strings.NewReader(line)) + csvRowReader.Comment = '#' + csvRowReader.Comma = ',' + + row, err := csvRowReader.Read() + if err != nil { + panic(fmt.Sprintf("failed parsing CSV line %s with erorr: %v\n", line, err)) + } + + return row +} diff --git a/scripts/degradation-tester/main_test.go b/scripts/degradation-tester/main_test.go index f93f1b8fa7..1204020d55 100644 --- a/scripts/degradation-tester/main_test.go +++ b/scripts/degradation-tester/main_test.go @@ -1,6 +1,7 @@ package main import ( + "fmt" "os" "testing" @@ -8,29 +9,21 @@ import ( ) func TestCheckLineWithValidData(t *testing.T) { - config := &Config{ - DefaultOpDelta: 10, - DefaultAllocDelta: 10, - } - line := "VerifyRSASignature-8,0.000266723,2%,0.0002658285,1%,~,p=0.123 n=10" + line := "VerifyRSASignature-8,0.000266723,2%" section := "sec/op" - errors := checkLine(config, line, section) - - require.Equal(t, 0, errors) + res := NewBenchmarkResult("pkg") + require.NoError(t, handleResultRow(line, section, res)) + require.Equal(t, 1, len(res.Res[section])) + require.Equal(t, 0.000266723, res.Res[section]["VerifyRSASignature"].Value) } func TestCheckLineWithInvalidData(t *testing.T) { - config := &Config{ - DefaultOpDelta: 10, - DefaultAllocDelta: 10, - } line := "VerifyRSASignature-8,0.000266723,2%,0.0002658285,1%,~,p=0.123 n=10" section := "invalid_section" - errors := checkLine(config, line, section) - - require.Equal(t, 1, errors) + res := NewBenchmarkResult("pkg") + require.Error(t, handleResultRow(line, section, res)) } func TestLoadConfigWithValidFile(t *testing.T) { @@ -48,8 +41,9 @@ func TestLoadConfigWithValidFile(t *testing.T) { require.NoError(t, err) require.NotNil(t, config) - require.Equal(t, 4.2, config.DefaultOpDelta) - require.Equal(t, 2.3, config.DefaultAllocDelta) + require.Equal(t, 4.2, config.DefaultSecOpDelta) + require.Equal(t, 1.0, config.DefaultBOpDelta) + require.Equal(t, 2.3, config.DefaultAllocOpDelta) } func TestLoadConfigWithInvalidFile(t *testing.T) { @@ -69,8 +63,9 @@ func TestNormalizeTestName(t *testing.T) { func TestGetThresholdForTestCase(t *testing.T) { config := &Config{ - DefaultOpDelta: 10, - DefaultAllocDelta: 10, + DefaultSecOpDelta: 10.0, + DefaultBOpDelta: 10.0, + DefaultAllocOpDelta: 10.0, } t.Run("TestGetThresholdForTestCaseWithDefaultSection", func(t *testing.T) { testName := "VerifyRSASignature" @@ -100,43 +95,221 @@ func TestParseBenchStatFile(t *testing.T) { f, err := os.Create(benchstatCsvFileName) require.NoError(t, err) - _, err = f.WriteString(stubBenchStatCSVFileData) + _, err = f.WriteString(benchstatResultOpenSSLUpgrade) require.NoError(t, err) cfg := &Config{ - DefaultAllocDelta: 10, - DefaultOpDelta: 10, + DefaultSecOpDelta: 10, + DefaultBOpDelta: 10, + DefaultAllocOpDelta: 10, } f, err = os.Open(benchstatCsvFileName) require.NoError(t, err) - totalErrors := checkFile(f, cfg) - require.Equal(t, 0, totalErrors) + checkResult := checkFile(f, cfg) + require.Equal(t, uint32(3), checkResult.TotalIssues) + for _, err := range checkResult.Issues[SectionTypeSecOp] { + fmt.Println(err) + } + for _, err := range checkResult.Issues[SectionTypeBop] { + fmt.Println(err) + } + for _, err := range checkResult.Issues[SectionTypeAllocOp] { + fmt.Println(err) + } } -const stubBenchStatCSVFileData = ` +var benchFast = ` goos: darwin +goarch: arm64 +pkg: github.com/bloxapp/ssv/scripts/degradation-tester +BenchmarkFastFunc-12 1 3001104583 ns/op 96 B/op 3 allocs/op +BenchmarkFastFunc-12 1 3000802917 ns/op 80 B/op 1 allocs/op +BenchmarkFastFunc-12 1 3001080000 ns/op 96 B/op 2 allocs/op +BenchmarkFastFunc-12 1 3000279167 ns/op 80 B/op 1 allocs/op +BenchmarkFastFunc-12 1 3001074167 ns/op 80 B/op 1 allocs/op +BenchmarkFastFunc-12 1 3000398125 ns/op 80 B/op 1 allocs/op +BenchmarkFastFunc-12 1 3001061375 ns/op 80 B/op 1 allocs/op +BenchmarkFastFunc-12 1 3001055667 ns/op 80 B/op 1 allocs/op +BenchmarkFastFunc-12 1 3001052167 ns/op 80 B/op 1 allocs/op +BenchmarkFastFunc-12 1 3000777625 ns/op 96 B/op 2 allocs/op +BenchmarkFastFunc-12 1 3000908750 ns/op 80 B/op 1 allocs/op +BenchmarkFastFunc-12 1 3001065125 ns/op 80 B/op 1 allocs/op +BenchmarkFastFunc-12 1 3001063541 ns/op 80 B/op 1 allocs/op +BenchmarkFastFunc-12 1 3000293458 ns/op 80 B/op 1 allocs/op +BenchmarkFastFunc-12 1 3001074041 ns/op 80 B/op 1 allocs/op +BenchmarkFastFunc-12 1 3001058000 ns/op 80 B/op 1 allocs/op +BenchmarkFastFunc-12 1 3000674750 ns/op 80 B/op 1 allocs/op +BenchmarkFastFunc-12 1 3000571333 ns/op 80 B/op 1 allocs/op +BenchmarkFastFunc-12 1 3000453709 ns/op 80 B/op 1 allocs/op +BenchmarkFastFunc-12 1 3001055292 ns/op 80 B/op 1 allocs/op +PASS +ok github.com/bloxapp/ssv/scripts/degradation-tester 60.693s +` + +var benchSlow = ` +goos: darwin +goarch: arm64 +pkg: github.com/bloxapp/ssv/scripts/degradation-tester +BenchmarkFastFunc-12 1 4001147542 ns/op 5512 B/op 7 allocs/op +BenchmarkFastFunc-12 1 3001054875 ns/op 96 B/op 2 allocs/op +BenchmarkFastFunc-12 1 4001054000 ns/op 88 B/op 2 allocs/op +BenchmarkFastFunc-12 1 3001111166 ns/op 80 B/op 1 allocs/op +BenchmarkFastFunc-12 1 4001046500 ns/op 80 B/op 1 allocs/op +BenchmarkFastFunc-12 1 5000376375 ns/op 80 B/op 1 allocs/op +BenchmarkFastFunc-12 1 5001059584 ns/op 80 B/op 1 allocs/op +BenchmarkFastFunc-12 1 5001063167 ns/op 80 B/op 1 allocs/op +BenchmarkFastFunc-12 1 3001043833 ns/op 80 B/op 1 allocs/op +BenchmarkFastFunc-12 1 3000403917 ns/op 80 B/op 1 allocs/op +BenchmarkFastFunc-12 1 3001043542 ns/op 96 B/op 2 allocs/op +BenchmarkFastFunc-12 1 3001066250 ns/op 80 B/op 1 allocs/op +BenchmarkFastFunc-12 1 4001067709 ns/op 80 B/op 1 allocs/op +BenchmarkFastFunc-12 1 5000513708 ns/op 80 B/op 1 allocs/op +BenchmarkFastFunc-12 1 3001065958 ns/op 80 B/op 1 allocs/op +BenchmarkFastFunc-12 1 5000536417 ns/op 80 B/op 1 allocs/op +BenchmarkFastFunc-12 1 4000695958 ns/op 80 B/op 1 allocs/op +BenchmarkFastFunc-12 1 3000862750 ns/op 80 B/op 1 allocs/op +BenchmarkFastFunc-12 1 3000408166 ns/op 80 B/op 1 allocs/op +BenchmarkFastFunc-12 1 3000444458 ns/op 80 B/op 1 allocs/op +PASS +ok github.com/bloxapp/ssv/scripts/degradation-tester 75.247s +` + +var benchstatResult = ` +goos: linux goarch: amd64 pkg: github.com/bloxapp/ssv/message/validation -cpu: xxxx -,./scripts/degradation-tester/benchmarks/message_validation_benchmarks_old.txt,,./scripts/degradation-tester/benchmarks/validation_results_new.txt -,sec/op,CI,sec/op,CI,vs base,P -VerifyRSASignature-8,0.000266723,2%,0.0002658285,1%,~,p=0.123 n=10 -geomean,0.0002667229999999998,,0.0002658284999999999,,-0.34% - -,./scripts/degradation-tester/benchmarks/message_validation_benchmarks_old.txt,,./scripts/degradation-tester/benchmarks/validation_results_new.txt -,B/op,CI,B/op,CI,vs base,P -VerifyRSASignature-8,1904.5,22%,1892.5,22%,~,p=0.927 n=10 -geomean,1904.4999999999998,,1892.4999999999995,,-0.63% - -,./scripts/degradation-tester/benchmarks/message_validation_benchmarks_old.txt,,./scripts/degradation-tester/benchmarks/validation_results_new.txt -,allocs/op,CI,allocs/op,CI,vs base,P -VerifyRSASignature-8,9.5,16%,9.5,16%,~,p=0.878 n=10 -geomean,9.500000000000002,,9.500000000000002,,+0.00% +cpu: AMD EPYC 7763 64-Core Processor +,scripts/degradation-tester/benchmarks/message_validation_benchmarks_old.txt +,sec/op,CI +VerifyRSASignature-4,0.00025610300000000004,1% +geomean,0.0002561029999999998 + +,scripts/degradation-tester/benchmarks/message_validation_benchmarks_old.txt +,B/op,CI +VerifyRSASignature-4,1872,22% +geomean,1871.9999999999993 + +,scripts/degradation-tester/benchmarks/message_validation_benchmarks_old.txt +,allocs/op,CI +VerifyRSASignature-4,9.5,16% +geomean,9.500000000000002 + +cpu: AMD EPYC 7763 64-Core Processor +,scripts/degradation-tester/benchmarks/message_validation_benchmarks_new.txt +,sec/op,CI +VerifyRSASignature-4,2.5498e-05,2% +geomean,2.5498e-05 + +,scripts/degradation-tester/benchmarks/message_validation_benchmarks_new.txt +,B/op,CI +VerifyRSASignature-4,187,28% +geomean,187.00000000000003 + +,scripts/degradation-tester/benchmarks/message_validation_benchmarks_new.txt +,allocs/op,CI +VerifyRSASignature-4,4,0% +geomean,4 + +` +var benchstatResult1 = ` +B31: summaries must be >0 to compute geomean +B45: summaries must be >0 to compute geomean +B74: summaries must be >0 to compute geomean +B88: summaries must be >0 to compute geomean +goos: linux +goarch: amd64 +pkg: github.com/bloxapp/ssv/protocol/v2/types +cpu: AMD EPYC 7763 64-Core Processor +,protocol_v2_types_benchmarks_old.txt +,sec/op,CI +VerifyPKCS1v15OpenSSL-4,2.3903500000000002e-05,1% +SignPKCS1v15OpenSSL-4,0.0006678130000000001,1% +VerifyBLS-4,0.001211613,1% +VerifyPKCS1v15-4,0.00025212400000000005,0% +VerifyPKCS1v15FastHash-4,0.00025184100000000004,0% +VerifyPSS-4,0.000255005,1% +SignBLS-4,0.0005117625,1% +SignPKCS1v15-4,0.0019599385,0% +SignPKCS1v15FastHash-4,0.0019667615,0% +SignPSS-4,0.001962333,0% +geomean,0.0005109289272306781 + +,protocol_v2_types_benchmarks_old.txt +,B/op,CI +VerifyPKCS1v15OpenSSL-4,40,0% +SignPKCS1v15OpenSSL-4,304,0% +VerifyBLS-4,0,0% +VerifyPKCS1v15-4,912,0% +VerifyPKCS1v15FastHash-4,912,0% +VerifyPSS-4,1120,0% +SignBLS-4,288,0% +SignPKCS1v15-4,896,0% +SignPKCS1v15FastHash-4,896,0% +SignPSS-4,1296,0% +geomean + +,protocol_v2_types_benchmarks_old.txt +,allocs/op,CI +VerifyPKCS1v15OpenSSL-4,3,0% +SignPKCS1v15OpenSSL-4,5,0% +VerifyBLS-4,0,0% +VerifyPKCS1v15-4,6,0% +VerifyPKCS1v15FastHash-4,6,0% +VerifyPSS-4,11,0% +SignBLS-4,1,0% +SignPKCS1v15-4,5,0% +SignPKCS1v15FastHash-4,5,0% +SignPSS-4,10,0% +geomean + +cpu: AMD EPYC 7763 64-Core Processor +,protocol_v2_types_benchmarks_new.txt +,sec/op,CI +VerifyPKCS1v15OpenSSL-4,2.3847s000000000004e-05,1% +SignPKCS1v15OpenSSL-4,0.0006664995,1% +VerifyBLS-4,0.001211802,0% +VerifyPKCS1v15-4,0.00025212050000000005,0% +VerifyPKCS1v15FastHash-4,0.00025198449999999997,0% +VerifyPSS-4,0.000254662,0% +SignBLS-4,0.0005147915000000001,1% +SignPKCS1v15-4,0.0019562055,0% +SignPKCS1v15FastHash-4,0.0019551660000000004,0% +SignPSS-4,0.0019617575000000003,0% +geomean,0.0005105621525118493 + +,protocol_v2_types_benchmarks_new.txt +,B/op,CI +VerifyPKCS1v15OpenSSL-4,40,0% +SignPKCS1v15OpenSSL-4,304,0% +VerifyBLS-4,0,0% +VerifyPKCS1v15-4,912,0% +VerifyPKCS1v15FastHash-4,912,0% +VerifyPSS-4,1120,0% +SignBLS-4,288,0% +SignPKCS1v15-4,896,0% +SignPKCS1v15FastHash-4,896,0% +SignPSS-4,1296,0% +geomean + +,protocol_v2_types_benchmarks_new.txt +,allocs/op,CI +VerifyPKCS1v15OpenSSL-4,3,0% +SignPKCS1v15OpenSSL-4,5,0% +VerifyBLS-4,0,0% +VerifyPKCS1v15-4,6,0% +VerifyPKCS1v15FastHash-4,6,0% +VerifyPSS-4,11,0% +SignBLS-4,1,0% +SignPKCS1v15-4,5,0% +SignPKCS1v15FastHash-4,5,0% +SignPSS-4,10,0% +geomean ` const stubConfig = ` -DefaultOpDelta: 4.2 -DefaultAllocDelta: 2.3 +DefaultSecOpDelta: 4.2 +DefaultBOpDelta: 1.0 +DefaultAllocOpDelta: 2.3 Packages: - Path: "./some/package/path" Tests: @@ -144,3 +317,40 @@ Packages: OpDelta: 10.0 AllocDelta: 4.5 ` + +var benchstatResultOpenSSLUpgrade = ` +goos: linux +goarch: amd64 +pkg: github.com/bloxapp/ssv/message/validation +cpu: AMD EPYC 7763 64-Core Processor +,old.txt +,sec/op,CI +VerifyRSASignature-4,0.00025610300000000004,1% +geomean,0.0002561029999999998 + +,old.txt +,B/op,CI +VerifyRSASignature-4,1872,22% +geomean,1871.9999999999993 + +,old.txt +,allocs/op,CI +VerifyRSASignature-4,9.5,16% +geomean,9.500000000000002 + +cpu: AMD EPYC 7763 64-Core Processor +,new.txt +,sec/op,CI +VerifyRSASignature-4,2.5280500000000003e-05,1% +geomean,2.528050000000001e-05 + +,new.txt +,B/op,CI +VerifyRSASignature-4,185.5,27% +geomean,185.49999999999994 + +,new.txt +,allocs/op,CI +VerifyRSASignature-4,4,0% +geomean,4 +` diff --git a/scripts/degradation-tester/types.go b/scripts/degradation-tester/types.go new file mode 100644 index 0000000000..31e2f62a5e --- /dev/null +++ b/scripts/degradation-tester/types.go @@ -0,0 +1,80 @@ +package main + +import "encoding/json" + +type Config struct { + DefaultSecOpDelta float64 `yaml:"DefaultSecOpDelta"` + DefaultBOpDelta float64 `yaml:"DefaultBOpDelta"` + DefaultAllocOpDelta float64 `yaml:"DefaultAllocOpDelta"` + Packages []BenchmarkingTestConfig `yaml:"Packages"` +} + +type BenchmarkingTestConfig struct { + Path string `yaml:"Path"` + Tests []TestCase `yaml:"Tests"` +} + +type TestCase struct { + Name string `yaml:"Name"` + OpDelta float64 `yaml:"OpDelta"` + AllocDelta float64 `yaml:"AllocDelta"` +} + +const ( + SectionTypeSecOp = "sec/op" + SectionTypeBop = "B/op" + SectionTypeAllocOp = "allocs/op" +) + +type TestResult struct { + Name string `json:"name"` + Value float64 `json:"value"` + //Value *big.Float `json:"value"` + CI string `json:"ci"` +} + +type BenchmarkResult struct { + Pkg string `json:"pkg"` + Res map[string]map[string]*TestResult `json:"res"` +} + +func (r *BenchmarkResult) MustMarshalJsonIndent() []byte { + jsonBytes, err := json.MarshalIndent(r, "", " ") + if err != nil { + panic(err) + } + return jsonBytes +} + +func NewBenchmarkResult(pkg string) *BenchmarkResult { + r := &BenchmarkResult{ + Pkg: pkg, + Res: make(map[string]map[string]*TestResult), + } + r.Res = make(map[string]map[string]*TestResult) + r.Res[SectionTypeSecOp] = make(map[string]*TestResult) + r.Res[SectionTypeBop] = make(map[string]*TestResult) + r.Res[SectionTypeAllocOp] = make(map[string]*TestResult) + return r +} + +type DegradationIssue struct { + TestName string + Threshold float64 + Diff float64 +} + +type DegradationCheckResult struct { + PkgName string + TotalIssues uint32 + Issues map[string][]*DegradationIssue +} + +func NewDegradationCheckResult(pkgName string) *DegradationCheckResult { + dc := &DegradationCheckResult{ + PkgName: pkgName, + Issues: make(map[string][]*DegradationIssue), + } + dc.Issues = make(map[string][]*DegradationIssue) + return dc +} diff --git a/scripts/degradation-tester/update-benchmarks.sh b/scripts/degradation-tester/update-benchmarks.sh index 05cf4eeb60..46faf6c718 100644 --- a/scripts/degradation-tester/update-benchmarks.sh +++ b/scripts/degradation-tester/update-benchmarks.sh @@ -10,7 +10,7 @@ for pkgPath in "${packagePaths[@]}"; do packageBenchName=$(echo "$pkgPath" | sed 's/\.\///g; s/\//_/g') benchmarksPath="${benchmarksResults}/${packageBenchName}_benchmarks_old.txt" - go test -bench=. -count=10 -benchmem "$pkgPath" | tee "$benchmarksPath" + go test -bench=. -count=16 -benchmem "$pkgPath" | tee "$benchmarksPath" echo "✅ Benchmarks updated for ${packageName} package." done