From 3b6366f52b2bd8a0957eabf821134a9429fe311a Mon Sep 17 00:00:00 2001 From: Anton Korpusenko Date: Thu, 29 Feb 2024 02:37:01 +0300 Subject: [PATCH] added workflow --- .github/workflows/degradation-test.yml | 35 +++++++ .../benchmarks/types_benchstat.txt | 43 --------- .../benchmarks/types_results_new.txt | 86 ------------------ .../benchmarks/validation_benchstat.txt | 15 --- .../benchmarks/validation_results_new.txt | 16 ---- scripts/degradation-tester/config.yaml | 10 +- .../degradation-tester/degradation-check.sh | 10 +- scripts/degradation-tester/degradation-tester | Bin 2880752 -> 2880752 bytes scripts/degradation-tester/main.go | 2 +- 9 files changed, 48 insertions(+), 169 deletions(-) create mode 100644 .github/workflows/degradation-test.yml delete mode 100644 scripts/degradation-tester/benchmarks/types_benchstat.txt delete mode 100644 scripts/degradation-tester/benchmarks/types_results_new.txt delete mode 100644 scripts/degradation-tester/benchmarks/validation_benchstat.txt delete mode 100644 scripts/degradation-tester/benchmarks/validation_results_new.txt diff --git a/.github/workflows/degradation-test.yml b/.github/workflows/degradation-test.yml new file mode 100644 index 0000000000..e4df0a92c0 --- /dev/null +++ b/.github/workflows/degradation-test.yml @@ -0,0 +1,35 @@ +name: Degradation test + +on: + push: + branches: + - "**" + +jobs: + align: + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: "1.20.x" + + - name: Tidy up dependencies + run: go mod tidy + + - name: Install yq + run: sudo snap install yq + + - name: Build degradation tester tool + run: | + cd ./scripts/degradation-tester + go install . + + - name: Run performance tests + run: | + make degradation-tester + env: + GO111MODULE: on diff --git a/scripts/degradation-tester/benchmarks/types_benchstat.txt b/scripts/degradation-tester/benchmarks/types_benchstat.txt deleted file mode 100644 index 75d5584b90..0000000000 --- a/scripts/degradation-tester/benchmarks/types_benchstat.txt +++ /dev/null @@ -1,43 +0,0 @@ -goos: darwin -goarch: amd64 -pkg: github.com/bloxapp/ssv/protocol/v2/types -cpu: Intel(R) Core(TM) i7-8700B CPU @ 3.20GHz - │ ./scripts/degradation-tester/benchmarks/types_results_old.txt │ ./scripts/degradation-tester/benchmarks/types_results_new.txt │ - │ sec/op │ sec/op vs base │ -VerifyBLS-12 987.4µ ± 2% 983.3µ ± 1% ~ (p=0.280 n=10) -VerifyPKCS1v15-12 212.0µ ± 2% 207.8µ ± 1% -1.96% (p=0.000 n=10) -VerifyPKCS1v15FastHash-12 215.2µ ± 11% 210.1µ ± 3% ~ (p=0.052 n=10) -VerifyPSS-12 218.0µ ± 4% 213.5µ ± 3% -2.07% (p=0.011 n=10) -SignBLS-12 407.2µ ± 2% 396.8µ ± 4% -2.54% (p=0.023 n=10) -SignPKCS1v15-12 1.426m ± 2% 1.435m ± 3% ~ (p=0.529 n=10) -SignPKCS1v15FastHash-12 1.411m ± 6% 1.429m ± 1% ~ (p=0.165 n=10) -SignPSS-12 1.444m ± 2% 1.427m ± 2% ~ (p=0.579 n=10) -geomean 573.0µ 566.7µ -1.09% - - │ ./scripts/degradation-tester/benchmarks/types_results_old.txt │ ./scripts/degradation-tester/benchmarks/types_results_new.txt │ - │ B/op │ B/op vs base │ -VerifyBLS-12 0.000 ± 0% 0.000 ± 0% ~ (p=1.000 n=10) ¹ -VerifyPKCS1v15-12 912.0 ± 0% 912.0 ± 0% ~ (p=1.000 n=10) ¹ -VerifyPKCS1v15FastHash-12 912.0 ± 0% 912.0 ± 0% ~ (p=1.000 n=10) ¹ -VerifyPSS-12 1.094Ki ± 0% 1.094Ki ± 0% ~ (p=1.000 n=10) ¹ -SignBLS-12 288.0 ± 0% 288.0 ± 0% ~ (p=1.000 n=10) ¹ -SignPKCS1v15-12 896.0 ± 0% 896.0 ± 0% ~ (p=1.000 n=10) ¹ -SignPKCS1v15FastHash-12 896.0 ± 0% 896.0 ± 0% ~ (p=1.000 n=10) ¹ -SignPSS-12 1.266Ki ± 0% 1.266Ki ± 0% ~ (p=1.000 n=10) ¹ -geomean ² +0.00% ² -¹ all samples are equal -² summaries must be >0 to compute geomean - - │ ./scripts/degradation-tester/benchmarks/types_results_old.txt │ ./scripts/degradation-tester/benchmarks/types_results_new.txt │ - │ allocs/op │ allocs/op vs base │ -VerifyBLS-12 0.000 ± 0% 0.000 ± 0% ~ (p=1.000 n=10) ¹ -VerifyPKCS1v15-12 6.000 ± 0% 6.000 ± 0% ~ (p=1.000 n=10) ¹ -VerifyPKCS1v15FastHash-12 6.000 ± 0% 6.000 ± 0% ~ (p=1.000 n=10) ¹ -VerifyPSS-12 11.00 ± 0% 11.00 ± 0% ~ (p=1.000 n=10) ¹ -SignBLS-12 1.000 ± 0% 1.000 ± 0% ~ (p=1.000 n=10) ¹ -SignPKCS1v15-12 5.000 ± 0% 5.000 ± 0% ~ (p=1.000 n=10) ¹ -SignPKCS1v15FastHash-12 5.000 ± 0% 5.000 ± 0% ~ (p=1.000 n=10) ¹ -SignPSS-12 10.00 ± 0% 10.00 ± 0% ~ (p=1.000 n=10) ¹ -geomean ² +0.00% ² -¹ all samples are equal -² summaries must be >0 to compute geomean diff --git a/scripts/degradation-tester/benchmarks/types_results_new.txt b/scripts/degradation-tester/benchmarks/types_results_new.txt deleted file mode 100644 index a5b13d5199..0000000000 --- a/scripts/degradation-tester/benchmarks/types_results_new.txt +++ /dev/null @@ -1,86 +0,0 @@ -goos: darwin -goarch: amd64 -pkg: github.com/bloxapp/ssv/protocol/v2/types -cpu: Intel(R) Core(TM) i7-8700B CPU @ 3.20GHz -BenchmarkVerifyBLS-12 1226 1001621 ns/op 0 B/op 0 allocs/op -BenchmarkVerifyBLS-12 1143 976704 ns/op 0 B/op 0 allocs/op -BenchmarkVerifyBLS-12 1237 994241 ns/op 0 B/op 0 allocs/op -BenchmarkVerifyBLS-12 1231 974793 ns/op 0 B/op 0 allocs/op -BenchmarkVerifyBLS-12 1231 995262 ns/op 0 B/op 0 allocs/op -BenchmarkVerifyBLS-12 1221 975624 ns/op 0 B/op 0 allocs/op -BenchmarkVerifyBLS-12 1224 987289 ns/op 0 B/op 0 allocs/op -BenchmarkVerifyBLS-12 1212 976047 ns/op 0 B/op 0 allocs/op -BenchmarkVerifyBLS-12 1200 979406 ns/op 0 B/op 0 allocs/op -BenchmarkVerifyBLS-12 1226 992229 ns/op 0 B/op 0 allocs/op -BenchmarkVerifyPKCS1v15-12 5870 206179 ns/op 912 B/op 6 allocs/op -BenchmarkVerifyPKCS1v15-12 5814 209133 ns/op 912 B/op 6 allocs/op -BenchmarkVerifyPKCS1v15-12 5690 206525 ns/op 912 B/op 6 allocs/op -BenchmarkVerifyPKCS1v15-12 5664 209075 ns/op 912 B/op 6 allocs/op -BenchmarkVerifyPKCS1v15-12 5787 206019 ns/op 912 B/op 6 allocs/op -BenchmarkVerifyPKCS1v15-12 5826 209231 ns/op 912 B/op 6 allocs/op -BenchmarkVerifyPKCS1v15-12 5788 206572 ns/op 912 B/op 6 allocs/op -BenchmarkVerifyPKCS1v15-12 5875 209176 ns/op 912 B/op 6 allocs/op -BenchmarkVerifyPKCS1v15-12 5715 210515 ns/op 912 B/op 6 allocs/op -BenchmarkVerifyPKCS1v15-12 5864 206448 ns/op 912 B/op 6 allocs/op -BenchmarkVerifyPKCS1v15FastHash-12 5745 209466 ns/op 912 B/op 6 allocs/op -BenchmarkVerifyPKCS1v15FastHash-12 5775 207003 ns/op 912 B/op 6 allocs/op -BenchmarkVerifyPKCS1v15FastHash-12 5704 214761 ns/op 912 B/op 6 allocs/op -BenchmarkVerifyPKCS1v15FastHash-12 5737 216464 ns/op 912 B/op 6 allocs/op -BenchmarkVerifyPKCS1v15FastHash-12 5761 242662 ns/op 912 B/op 6 allocs/op -BenchmarkVerifyPKCS1v15FastHash-12 5637 214257 ns/op 912 B/op 6 allocs/op -BenchmarkVerifyPKCS1v15FastHash-12 5732 210811 ns/op 912 B/op 6 allocs/op -BenchmarkVerifyPKCS1v15FastHash-12 5058 208281 ns/op 912 B/op 6 allocs/op -BenchmarkVerifyPKCS1v15FastHash-12 5778 207746 ns/op 912 B/op 6 allocs/op -BenchmarkVerifyPKCS1v15FastHash-12 5810 208474 ns/op 912 B/op 6 allocs/op -BenchmarkVerifyPSS-12 5641 219411 ns/op 1120 B/op 11 allocs/op -BenchmarkVerifyPSS-12 5726 213746 ns/op 1120 B/op 11 allocs/op -BenchmarkVerifyPSS-12 5785 213584 ns/op 1120 B/op 11 allocs/op -BenchmarkVerifyPSS-12 5673 219355 ns/op 1120 B/op 11 allocs/op -BenchmarkVerifyPSS-12 5797 211165 ns/op 1120 B/op 11 allocs/op -BenchmarkVerifyPSS-12 5710 213255 ns/op 1120 B/op 11 allocs/op -BenchmarkVerifyPSS-12 5785 213406 ns/op 1120 B/op 11 allocs/op -BenchmarkVerifyPSS-12 5679 215288 ns/op 1120 B/op 11 allocs/op -BenchmarkVerifyPSS-12 5756 210423 ns/op 1120 B/op 11 allocs/op -BenchmarkVerifyPSS-12 5269 210196 ns/op 1120 B/op 11 allocs/op -BenchmarkSignBLS-12 3026 397140 ns/op 288 B/op 1 allocs/op -BenchmarkSignBLS-12 3032 390826 ns/op 288 B/op 1 allocs/op -BenchmarkSignBLS-12 3027 390042 ns/op 288 B/op 1 allocs/op -BenchmarkSignBLS-12 3081 411725 ns/op 288 B/op 1 allocs/op -BenchmarkSignBLS-12 3032 411257 ns/op 288 B/op 1 allocs/op -BenchmarkSignBLS-12 3025 396492 ns/op 288 B/op 1 allocs/op -BenchmarkSignBLS-12 3128 398866 ns/op 288 B/op 1 allocs/op -BenchmarkSignBLS-12 3038 395501 ns/op 288 B/op 1 allocs/op -BenchmarkSignBLS-12 3022 407281 ns/op 288 B/op 1 allocs/op -BenchmarkSignBLS-12 3090 392919 ns/op 288 B/op 1 allocs/op -BenchmarkSignPKCS1v15-12 853 1445515 ns/op 896 B/op 5 allocs/op -BenchmarkSignPKCS1v15-12 844 1403513 ns/op 896 B/op 5 allocs/op -BenchmarkSignPKCS1v15-12 859 1407298 ns/op 896 B/op 5 allocs/op -BenchmarkSignPKCS1v15-12 844 1403500 ns/op 896 B/op 5 allocs/op -BenchmarkSignPKCS1v15-12 852 1531021 ns/op 896 B/op 5 allocs/op -BenchmarkSignPKCS1v15-12 835 1455804 ns/op 896 B/op 5 allocs/op -BenchmarkSignPKCS1v15-12 852 1424019 ns/op 896 B/op 5 allocs/op -BenchmarkSignPKCS1v15-12 813 1476737 ns/op 896 B/op 5 allocs/op -BenchmarkSignPKCS1v15-12 862 1450105 ns/op 896 B/op 5 allocs/op -BenchmarkSignPKCS1v15-12 834 1408614 ns/op 896 B/op 5 allocs/op -BenchmarkSignPKCS1v15FastHash-12 838 1434899 ns/op 896 B/op 5 allocs/op -BenchmarkSignPKCS1v15FastHash-12 860 1413496 ns/op 896 B/op 5 allocs/op -BenchmarkSignPKCS1v15FastHash-12 849 1441625 ns/op 896 B/op 5 allocs/op -BenchmarkSignPKCS1v15FastHash-12 849 1500328 ns/op 896 B/op 5 allocs/op -BenchmarkSignPKCS1v15FastHash-12 847 1433547 ns/op 896 B/op 5 allocs/op -BenchmarkSignPKCS1v15FastHash-12 781 1420704 ns/op 896 B/op 5 allocs/op -BenchmarkSignPKCS1v15FastHash-12 850 1412588 ns/op 896 B/op 5 allocs/op -BenchmarkSignPKCS1v15FastHash-12 846 1424265 ns/op 896 B/op 5 allocs/op -BenchmarkSignPKCS1v15FastHash-12 828 1437411 ns/op 896 B/op 5 allocs/op -BenchmarkSignPKCS1v15FastHash-12 832 1404216 ns/op 896 B/op 5 allocs/op -BenchmarkSignPSS-12 717 1413495 ns/op 1296 B/op 10 allocs/op -BenchmarkSignPSS-12 848 1409720 ns/op 1296 B/op 10 allocs/op -BenchmarkSignPSS-12 848 1426474 ns/op 1296 B/op 10 allocs/op -BenchmarkSignPSS-12 808 1448838 ns/op 1296 B/op 10 allocs/op -BenchmarkSignPSS-12 838 1459550 ns/op 1296 B/op 10 allocs/op -BenchmarkSignPSS-12 844 1526626 ns/op 1296 B/op 10 allocs/op -BenchmarkSignPSS-12 819 1427420 ns/op 1296 B/op 10 allocs/op -BenchmarkSignPSS-12 837 1425498 ns/op 1296 B/op 10 allocs/op -BenchmarkSignPSS-12 852 1421791 ns/op 1296 B/op 10 allocs/op -BenchmarkSignPSS-12 846 1458324 ns/op 1296 B/op 10 allocs/op -PASS -ok github.com/bloxapp/ssv/protocol/v2/types 104.703s diff --git a/scripts/degradation-tester/benchmarks/validation_benchstat.txt b/scripts/degradation-tester/benchmarks/validation_benchstat.txt deleted file mode 100644 index d568fa3d71..0000000000 --- a/scripts/degradation-tester/benchmarks/validation_benchstat.txt +++ /dev/null @@ -1,15 +0,0 @@ -goos: darwin -goarch: amd64 -pkg: github.com/bloxapp/ssv/message/validation -cpu: Intel(R) Core(TM) i7-8700B CPU @ 3.20GHz - │ ./scripts/degradation-tester/benchmarks/validation_results_old.txt │ ./scripts/degradation-tester/benchmarks/validation_results_new.txt │ - │ sec/op │ sec/op vs base │ -VerifyRSASignature-12 207.7µ ± 0% 210.3µ ± 4% ~ (p=0.052 n=10) - - │ ./scripts/degradation-tester/benchmarks/validation_results_old.txt │ ./scripts/degradation-tester/benchmarks/validation_results_new.txt │ - │ B/op │ B/op vs base │ -VerifyRSASignature-12 1.642Ki ± 19% 1.638Ki ± 18% ~ (p=1.000 n=10) - - │ ./scripts/degradation-tester/benchmarks/validation_results_old.txt │ ./scripts/degradation-tester/benchmarks/validation_results_new.txt │ - │ allocs/op │ allocs/op vs base │ -VerifyRSASignature-12 8.500 ± 18% 8.500 ± 18% ~ (p=1.000 n=10) diff --git a/scripts/degradation-tester/benchmarks/validation_results_new.txt b/scripts/degradation-tester/benchmarks/validation_results_new.txt deleted file mode 100644 index 1ed0cb84ec..0000000000 --- a/scripts/degradation-tester/benchmarks/validation_results_new.txt +++ /dev/null @@ -1,16 +0,0 @@ -goos: darwin -goarch: amd64 -pkg: github.com/bloxapp/ssv/message/validation -cpu: Intel(R) Core(TM) i7-8700B CPU @ 3.20GHz -BenchmarkVerifyRSASignature-12 5348 207648 ns/op 1270 B/op 7 allocs/op -BenchmarkVerifyRSASignature-12 5785 230800 ns/op 1398 B/op 7 allocs/op -BenchmarkVerifyRSASignature-12 5862 218655 ns/op 1488 B/op 8 allocs/op -BenchmarkVerifyRSASignature-12 5775 211573 ns/op 1556 B/op 8 allocs/op -BenchmarkVerifyRSASignature-12 5868 206721 ns/op 1634 B/op 8 allocs/op -BenchmarkVerifyRSASignature-12 5265 212132 ns/op 1720 B/op 9 allocs/op -BenchmarkVerifyRSASignature-12 5810 208004 ns/op 1816 B/op 9 allocs/op -BenchmarkVerifyRSASignature-12 5800 209013 ns/op 1910 B/op 9 allocs/op -BenchmarkVerifyRSASignature-12 5563 207900 ns/op 1986 B/op 10 allocs/op -BenchmarkVerifyRSASignature-12 5551 212551 ns/op 2113 B/op 10 allocs/op -PASS -ok github.com/bloxapp/ssv/message/validation 34.427s diff --git a/scripts/degradation-tester/config.yaml b/scripts/degradation-tester/config.yaml index 1c1eab7969..c148242223 100644 --- a/scripts/degradation-tester/config.yaml +++ b/scripts/degradation-tester/config.yaml @@ -6,17 +6,17 @@ Tests: - PackagePath: "./protocol/v2/types" TestCases: - Name: "VerifyBLS" - AllowedDeltaPercentage: 3.0 + AllowedDeltaPercentage: 6.0 - Name: "VerifyPKCS1v15" AllowedDeltaPercentage: 3.0 - Name: "VerifyPKCS1v15FastHash" - AllowedDeltaPercentage: 3.0 + AllowedDeltaPercentage: 6.0 - Name: "VerifyPSS" - AllowedDeltaPercentage: 3.0 + AllowedDeltaPercentage: 6.0 - Name: "SignBLS" - AllowedDeltaPercentage: 3.0 + AllowedDeltaPercentage: 6.0 - Name: "SignPKCS1v15" - AllowedDeltaPercentage: 3.0 + AllowedDeltaPercentage: 6.0 - Name: "SignPKCS1v15FastHash" AllowedDeltaPercentage: 3.0 - Name: "SignPSS" diff --git a/scripts/degradation-tester/degradation-check.sh b/scripts/degradation-tester/degradation-check.sh index 8241c6bb24..eaa32a2a69 100644 --- a/scripts/degradation-tester/degradation-check.sh +++ b/scripts/degradation-tester/degradation-check.sh @@ -1,5 +1,6 @@ #!/bin/bash + prefix="./scripts/degradation-tester" configFile="$prefix/config.yaml" benchmarksResults="$prefix/benchmarks" @@ -10,17 +11,20 @@ for pkgPath in "${packagePaths[@]}"; do packageName=$(basename "$pkgPath") outputFile="${benchmarksResults}/${packageName}_results_new.txt" oldBenchmarks="${benchmarksResults}/${packageName}_results_old.txt" + benchStatFile="${benchmarksResults}/${packageName}_benchstat.txt" -# echo "Benchmarking package $packageName..." + go test -bench=. -count=10 -benchmem "$pkgPath" | tee "$outputFile" - benchStatFile="${benchmarksResults}/${packageName}_benchstat.txt" benchstat "$oldBenchmarks" "$outputFile" &> "${benchStatFile}" - "${prefix}/degradation-tester" "${configFile}" "${benchStatFile}" + degradation-tester "${configFile}" "${benchStatFile}" if [ $? -ne 0 ]; then echo "❌ Degradation tests have failed for ${packageName} package." exit 1 fi echo "✅ Degradation tests have passed for ${packageName} package." + + rm "${benchStatFile}" + rm "${outputFile}" done diff --git a/scripts/degradation-tester/degradation-tester b/scripts/degradation-tester/degradation-tester index 4a723dcd1119b81e52df470f6b99214c2beecb85..150c2684e1c5dcc16783d5ab249e2779300ee873 100755 GIT binary patch delta 820 zcmbWx%TE(g6u{v*v@$S)bQt6%733iTLK#Yd!3gE)1PBlcs0axoZE2AfC{;>?P{c4) zQxr7Zuu@};4@fA%ctuvk4J#xXk%Smc*l65fgpDrnqKSWii*Iwz$uDR7>l|zoK9#D` zZs~V88gdPeqRTlJdwp4Plc8Fpt8}=Uy3h2NG>9%&h0{=9D_qgoEMi+#u~n$ys||hS zPDgf0ok3%5tJKw1ovk%jxp-S?XMI(+)f~^n-O*=8FSu318Yo=w)4RXJC?4VAQ6YRl zDLpXdM}^|$9+(hd2ODPKeKl<0l%Jru0JAu~!Ee_E1#-fwA5K_R?5JP{M}AWLT7%Pa zm`_O@z+pcBNef9;7kOB3Tn zuexY|<*n6=UR@(%d`v<>=%0x$4e5MLIVjP9k2wV^=%tT|O@H$C-}T`7eskR!BYM_E z&yPh4NIA*$w;V~z1F~|S!OHoA+F7WO^IC#%UQ2NID{jZ4phS-t@Qo9_t^m;+yk(J> zA`THDgbFW3uum=czMJiUsD@st-IO0^G_^Ya>aQP+mf zPjhNID_@(F&kOS4G)IIHhly|^f`}xdh-f@GeWZ2eztV`LMXWN!+=fwO%nPHswGST1 k(x#d?O2iUz#4#eCNFWl4B;q)cOq?K6h*T_XrrnDA4Y>Cw6951J delta 818 zcmbWx-Ahw(7{Kv8+w#nl)=tNquWdQaW|}+d(al~o6*)<2q_b&>?!_%%r&IGSai(}& zG0PSG@JhO3P| zbf3*P6`X8L&1|c(rWd#7HP>|Jm7b`t>vCT-$I+S_x%#MCk8&^-j?GGQJzrrM4{>lO z2bPhp4Q@$)F?gX3#*Od|mpy>x5?H~ypCCI2YdFOu-ROp1QM)L`+C|w9B~0On_p;3; z=n=yLa$*M#3k3ODh{pGx2My0d14wtb7gVNpi?c3U<8*1bWi99c!kCr*-6-mxF)tVm|TpTECs-nk*h_>HDk6f2B%*NtM0D-T|4Jj07O=7`<~oenV; ") + fmt.Println("Usage: degradation-tester ") os.Exit(1) }