From 3ffcce1cc9090d8eebd7faa902846f8a5da8ec3e Mon Sep 17 00:00:00 2001 From: Tim Ross Date: Fri, 15 Nov 2024 13:59:54 -0500 Subject: [PATCH] Increase the number of times CI runs benchmarks This is in an attempt to provide more reliable test results. 6 was chosen as that's the minimum number benchstat requires for a 95% confidence interval. --- .github/workflows/benchmark-code-nonroot.yaml | 2 +- .github/workflows/benchmark-code-root.yaml | 2 +- Makefile | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/benchmark-code-nonroot.yaml b/.github/workflows/benchmark-code-nonroot.yaml index 1df6ebacdd22b..40c59c362b2c5 100644 --- a/.github/workflows/benchmark-code-nonroot.yaml +++ b/.github/workflows/benchmark-code-nonroot.yaml @@ -59,7 +59,7 @@ jobs: # Run benchmarks once to make sure they don't break # Must be run separate since gotestsum is not compatible with benchmark output - name: Run Benchmarks Once - timeout-minutes: 5 + timeout-minutes: 25 shell: bash # Overriding default shell which is `sh -e` run: make test-go-bench | sed -u -E "s/^(FAIL\s+github)/::error title=Benchmark Failed::\1/" diff --git a/.github/workflows/benchmark-code-root.yaml b/.github/workflows/benchmark-code-root.yaml index 72fa3065f584f..99062d48285dd 100644 --- a/.github/workflows/benchmark-code-root.yaml +++ b/.github/workflows/benchmark-code-root.yaml @@ -59,7 +59,7 @@ jobs: # Run benchmarks once to make sure they don't break # Must be run separate since gotestsum is not compatible with benchmark output - name: Run Benchmarks Once - timeout-minutes: 5 + timeout-minutes: 25 shell: bash # Overriding default shell which is `sh -e` run: make test-go-bench-root | sed -u -E "s/^(FAIL\s+github)/::error title=Benchmark Failed::\1/" diff --git a/Makefile b/Makefile index 7012ac04e823d..773593bcf3b35 100644 --- a/Makefile +++ b/Makefile @@ -943,14 +943,14 @@ endif test-go-bench: PACKAGES = $(shell grep --exclude-dir api --include "*_test.go" -lr testing.B . | xargs dirname | xargs go list | sort -u) test-go-bench: BENCHMARK_SKIP_PATTERN = "^BenchmarkRoot" test-go-bench: | $(TEST_LOG_DIR) - go test -run ^$$ -bench . -skip $(BENCHMARK_SKIP_PATTERN) -benchtime 1x $(PACKAGES) \ + go test -run ^$$ -bench . -skip $(BENCHMARK_SKIP_PATTERN) -count=6 -benchtime 1x $(PACKAGES) \ | tee $(TEST_LOG_DIR)/bench.txt test-go-bench-root: PACKAGES = $(shell grep --exclude-dir api --include "*_test.go" -lr BenchmarkRoot . | xargs dirname | xargs go list | sort -u) test-go-bench-root: BENCHMARK_PATTERN = "^BenchmarkRoot" test-go-bench-root: BENCHMARK_SKIP_PATTERN = "" test-go-bench-root: | $(TEST_LOG_DIR) - go test -run ^$$ -bench $(BENCHMARK_PATTERN) -skip $(BENCHMARK_SKIP_PATTERN) -benchtime 1x $(PACKAGES) \ + go test -run ^$$ -bench $(BENCHMARK_PATTERN) -skip $(BENCHMARK_SKIP_PATTERN) -count=6 -benchtime 1x $(PACKAGES) \ | tee $(TEST_LOG_DIR)/bench.txt # Make sure untagged vnetdaemon code build/tests.