From 33223eb62c1e106debddd3f2ac15abc46a02dbdd Mon Sep 17 00:00:00 2001 From: dantecatalfamo Date: Wed, 8 Jan 2025 16:18:25 -0500 Subject: [PATCH 1/6] Try splitting tests up --- .github/workflows/test-go.yaml | 14 +++++++++----- Makefile | 30 +++++++++++++++++++++++------- 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/.github/workflows/test-go.yaml b/.github/workflows/test-go.yaml index 75c480092854..162f4f54291d 100644 --- a/.github/workflows/test-go.yaml +++ b/.github/workflows/test-go.yaml @@ -42,7 +42,7 @@ jobs: test-go: strategy: matrix: - suite: ["integration", "core"] + suite: ["integration", "core", "migration", "fleetctl", "vuln"] os: [ubuntu-latest] mysql: ["mysql:8.0.36", "mysql:8.4.3", "mysql:9.1.0"] # make sure to update supported versions docs when this changes isCron: @@ -118,11 +118,15 @@ jobs: - name: Run Go Tests run: | if [[ "${{ matrix.suite }}" == "core" ]]; then - RUN_TESTS_ARG='-skip=^TestIntegrations' + CI_TEST_PKG=main RUN_TESTS_ARG='-skip=^TestIntegrations' elif [[ "${{ matrix.suite }}" == "integration" ]]; then - RUN_TESTS_ARG='-run=^TestIntegrations' - else - RUN_TESTS_ARG='' + CI_TEST_PKG=main RUN_TESTS_ARG='-run=^TestIntegrations' + elif [[ "${{ matrix.suite }}" == "fleetctl" ]]; then + CI_TEST_PKG=fleetctl RUN_TESTS_ARG='' + elif [[ "${{ matrix.suite }}" == "migration" ]]; then + CI_TEST_PKG=migration RUN_TESTS_ARG='' + elif [[ "${{ matrix.suite }}" == "vuln" ]]; then + CI_TEST_PKG=vuln RUN_TESTS_ARG='' fi GO_TEST_EXTRA_FLAGS="-v -race=$RACE_ENABLED -timeout=$GO_TEST_TIMEOUT $RUN_TESTS_ARG" \ TEST_LOCK_FILE_PATH=$(pwd)/lock \ diff --git a/Makefile b/Makefile index 406d8486bf79..2979acca793a 100644 --- a/Makefile +++ b/Makefile @@ -150,14 +150,30 @@ dump-test-schema: # TESTS_TO_RUN: Name specific tests to run in the specified packages. Leave blank to run all tests in the specified packages. # GO_TEST_EXTRA_FLAGS: Used to specify other arguments to `go test`. # GO_TEST_MAKE_FLAGS: Internal var used by other targets to add arguments to `go test`. -# +# PKG_TO_TEST := "" # default to empty string; can be overridden on command line. go_test_pkg_to_test := $(addprefix ./,$(PKG_TO_TEST)) # set paths for packages to test dlv_test_pkg_to_test := $(addprefix github.com/fleetdm/fleet/v4/,$(PKG_TO_TEST)) # set URIs for packages to debug +DEFAULT_PKG_TO_TEST := ./cmd/... ./ee/... ./orbit/pkg/... ./orbit/cmd/orbit ./pkg/... ./server/... ./tools/... +ifeq ($(CI_TEST_PKG), main) + CI_PKG_TO_TEST=$(shell go list ${DEFAULT_PKG_TO_TEST} | sed -e 's|github.com/fleetdm/fleet/v4/||g') +else ifeq ($(CI_TEST_PKG), migration) + CI_PKG_TO_TEST="server/datastore/mysql/migrations/..." +else ifeq ($(CI_TEST_PKG), fleetctl) + CI_PKG_TO_TEST="cmd/fleetctl/..." +else ifeq ($(CI_TEST_PKG), vuln) + CI_PKG_TO_TEST="server/vulnerabilities/..." +else + CI_PKG_TO_TEST=$(DEFAULT_PKG_TO_TEST) +endif + +ci-pkg: + @echo $(CI_PKG_TO_TEST) + .run-go-tests: ifeq ($(PKG_TO_TEST), "") - @echo "Please specify one or more packages to test with argument PKG_TO_TEST=\"/path/to/pkg/1 /path/to/pkg/2\"..."; + @echo "Please specify one or more packages to test with argument PKG_TO_TEST=\"/path/to/pkg/1 /path/to/pkg/2\"..."; else @echo Running Go tests with command: go test -tags full,fts5,netgo -run=${TESTS_TO_RUN} ${GO_TEST_MAKE_FLAGS} ${GO_TEST_EXTRA_FLAGS} -parallel 8 -coverprofile=coverage.txt -covermode=atomic -coverpkg=github.com/fleetdm/fleet/v4/... $(go_test_pkg_to_test) @@ -171,10 +187,10 @@ endif # GO_TEST_EXTRA_FLAGS: Used to specify other arguments to `go test`. .debug-go-tests: ifeq ($(PKG_TO_TEST), "") - @echo "Please specify one or more packages to debug with argument PKG_TO_TEST=\"/path/to/pkg/1 /path/to/pkg/2\"..."; + @echo "Please specify one or more packages to debug with argument PKG_TO_TEST=\"/path/to/pkg/1 /path/to/pkg/2\"..."; else @echo Debugging tests with command: - dlv test ${dlv_test_pkg_to_test} --api-version=2 --listen=127.0.0.1:61179 ${DEBUG_TEST_EXTRA_FLAGS} -- -test.v -test.run=${TESTS_TO_RUN} ${GO_TEST_EXTRA_FLAGS} + dlv test ${dlv_test_pkg_to_test} --api-version=2 --listen=127.0.0.1:61179 ${DEBUG_TEST_EXTRA_FLAGS} -- -test.v -test.run=${TESTS_TO_RUN} ${GO_TEST_EXTRA_FLAGS} endif # Command to run specific tests in development. Can run all tests for one or more packages, or specific tests within packages. @@ -182,11 +198,11 @@ run-go-tests: @MYSQL_TEST=1 REDIS_TEST=1 MINIO_STORAGE_TEST=1 SAML_IDP_TEST=1 NETWORK_TEST=1 make .run-go-tests GO_TEST_MAKE_FLAGS="-v" debug-go-tests: - @MYSQL_TEST=1 REDIS_TEST=1 MINIO_STORAGE_TEST=1 SAML_IDP_TEST=1 NETWORK_TEST=1 make .debug-go-tests + @MYSQL_TEST=1 REDIS_TEST=1 MINIO_STORAGE_TEST=1 SAML_IDP_TEST=1 NETWORK_TEST=1 make .debug-go-tests # Command used in CI to run all tests. -test-go: dump-test-schema generate-mock - make .run-go-tests PKG_TO_TEST="./cmd/... ./ee/... ./orbit/pkg/... ./orbit/cmd/orbit ./pkg/... ./server/... ./tools/..." +test-go: # dump-test-schema generate-mock + make .run-go-tests PKG_TO_TEST="$(CI_PKG_TO_TEST)" analyze-go: go test -tags full,fts5,netgo -race -cover ./... From 7f768c7886f72692307ee4c3fbb78da2aec5ed7c Mon Sep 17 00:00:00 2001 From: dantecatalfamo Date: Wed, 8 Jan 2025 16:22:37 -0500 Subject: [PATCH 2/6] Arg wasn't in the env --- .github/workflows/test-go.yaml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-go.yaml b/.github/workflows/test-go.yaml index 162f4f54291d..d525ce7c8dab 100644 --- a/.github/workflows/test-go.yaml +++ b/.github/workflows/test-go.yaml @@ -118,15 +118,20 @@ jobs: - name: Run Go Tests run: | if [[ "${{ matrix.suite }}" == "core" ]]; then - CI_TEST_PKG=main RUN_TESTS_ARG='-skip=^TestIntegrations' + CI_TEST_PKG=main + RUN_TESTS_ARG='-skip=^TestIntegrations' elif [[ "${{ matrix.suite }}" == "integration" ]]; then - CI_TEST_PKG=main RUN_TESTS_ARG='-run=^TestIntegrations' + CI_TEST_PKG=main + RUN_TESTS_ARG='-run=^TestIntegrations' elif [[ "${{ matrix.suite }}" == "fleetctl" ]]; then - CI_TEST_PKG=fleetctl RUN_TESTS_ARG='' + CI_TEST_PKG=fleetctl + RUN_TESTS_ARG='' elif [[ "${{ matrix.suite }}" == "migration" ]]; then - CI_TEST_PKG=migration RUN_TESTS_ARG='' + CI_TEST_PKG=migration + RUN_TESTS_ARG='' elif [[ "${{ matrix.suite }}" == "vuln" ]]; then - CI_TEST_PKG=vuln RUN_TESTS_ARG='' + CI_TEST_PKG=vuln + RUN_TESTS_ARG='' fi GO_TEST_EXTRA_FLAGS="-v -race=$RACE_ENABLED -timeout=$GO_TEST_TIMEOUT $RUN_TESTS_ARG" \ TEST_LOCK_FILE_PATH=$(pwd)/lock \ @@ -139,6 +144,7 @@ jobs: SAML_IDP_TEST=1 \ MAIL_TEST=1 \ NETWORK_TEST_GITHUB_TOKEN=${{ secrets.FLEET_RELEASE_GITHUB_PAT }} \ + CI_TEST_PKG="$(CI_TEST_PKG)" \ make test-go 2>&1 | tee /tmp/gotest.log - name: Create mysql identifier without colon From 5c347dbfeaab22226e2efd64715ebd8e6fdef5da Mon Sep 17 00:00:00 2001 From: dantecatalfamo Date: Wed, 8 Jan 2025 16:25:46 -0500 Subject: [PATCH 3/6] Used make var in script --- .github/workflows/test-go.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-go.yaml b/.github/workflows/test-go.yaml index d525ce7c8dab..36328d40d1a4 100644 --- a/.github/workflows/test-go.yaml +++ b/.github/workflows/test-go.yaml @@ -144,7 +144,7 @@ jobs: SAML_IDP_TEST=1 \ MAIL_TEST=1 \ NETWORK_TEST_GITHUB_TOKEN=${{ secrets.FLEET_RELEASE_GITHUB_PAT }} \ - CI_TEST_PKG="$(CI_TEST_PKG)" \ + CI_TEST_PKG="${CI_TEST_PKG}" \ make test-go 2>&1 | tee /tmp/gotest.log - name: Create mysql identifier without colon From 53a64373e89006e38baf7daec65559751617ac8c Mon Sep 17 00:00:00 2001 From: dantecatalfamo Date: Wed, 8 Jan 2025 16:51:28 -0500 Subject: [PATCH 4/6] Forgot to remove tests that were split off --- .github/workflows/test-go.yaml | 10 ++-------- Makefile | 4 ++-- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test-go.yaml b/.github/workflows/test-go.yaml index 36328d40d1a4..1be063dcc86d 100644 --- a/.github/workflows/test-go.yaml +++ b/.github/workflows/test-go.yaml @@ -123,14 +123,8 @@ jobs: elif [[ "${{ matrix.suite }}" == "integration" ]]; then CI_TEST_PKG=main RUN_TESTS_ARG='-run=^TestIntegrations' - elif [[ "${{ matrix.suite }}" == "fleetctl" ]]; then - CI_TEST_PKG=fleetctl - RUN_TESTS_ARG='' - elif [[ "${{ matrix.suite }}" == "migration" ]]; then - CI_TEST_PKG=migration - RUN_TESTS_ARG='' - elif [[ "${{ matrix.suite }}" == "vuln" ]]; then - CI_TEST_PKG=vuln + else + CI_TEST_PKG="${{ matrix.suite }}" RUN_TESTS_ARG='' fi GO_TEST_EXTRA_FLAGS="-v -race=$RACE_ENABLED -timeout=$GO_TEST_TIMEOUT $RUN_TESTS_ARG" \ diff --git a/Makefile b/Makefile index 2979acca793a..b91184f538bc 100644 --- a/Makefile +++ b/Makefile @@ -157,7 +157,7 @@ dlv_test_pkg_to_test := $(addprefix github.com/fleetdm/fleet/v4/,$(PKG_TO_TEST)) DEFAULT_PKG_TO_TEST := ./cmd/... ./ee/... ./orbit/pkg/... ./orbit/cmd/orbit ./pkg/... ./server/... ./tools/... ifeq ($(CI_TEST_PKG), main) - CI_PKG_TO_TEST=$(shell go list ${DEFAULT_PKG_TO_TEST} | sed -e 's|github.com/fleetdm/fleet/v4/||g') + CI_PKG_TO_TEST=$(shell go list ${DEFAULT_PKG_TO_TEST} | grep -v "server/datastore/mysql/migrations" | grep -v "cmd/fleetctl" | grep -v "server/vulnerabilities" | sed -e 's|github.com/fleetdm/fleet/v4/||g') else ifeq ($(CI_TEST_PKG), migration) CI_PKG_TO_TEST="server/datastore/mysql/migrations/..." else ifeq ($(CI_TEST_PKG), fleetctl) @@ -168,7 +168,7 @@ else CI_PKG_TO_TEST=$(DEFAULT_PKG_TO_TEST) endif -ci-pkg: +ci-pkg-list: @echo $(CI_PKG_TO_TEST) .run-go-tests: From a8e06b1042749e1b8d685cbf237e733c9fc6a5b8 Mon Sep 17 00:00:00 2001 From: dantecatalfamo Date: Wed, 8 Jan 2025 17:12:29 -0500 Subject: [PATCH 5/6] Try moving all mysql tests out, not just migrations --- .github/workflows/test-go.yaml | 2 +- Makefile | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-go.yaml b/.github/workflows/test-go.yaml index 1be063dcc86d..e347f7278217 100644 --- a/.github/workflows/test-go.yaml +++ b/.github/workflows/test-go.yaml @@ -42,7 +42,7 @@ jobs: test-go: strategy: matrix: - suite: ["integration", "core", "migration", "fleetctl", "vuln"] + suite: ["integration", "core", "mysql", "fleetctl", "vuln"] os: [ubuntu-latest] mysql: ["mysql:8.0.36", "mysql:8.4.3", "mysql:9.1.0"] # make sure to update supported versions docs when this changes isCron: diff --git a/Makefile b/Makefile index b91184f538bc..fa8ec9dac69f 100644 --- a/Makefile +++ b/Makefile @@ -157,9 +157,9 @@ dlv_test_pkg_to_test := $(addprefix github.com/fleetdm/fleet/v4/,$(PKG_TO_TEST)) DEFAULT_PKG_TO_TEST := ./cmd/... ./ee/... ./orbit/pkg/... ./orbit/cmd/orbit ./pkg/... ./server/... ./tools/... ifeq ($(CI_TEST_PKG), main) - CI_PKG_TO_TEST=$(shell go list ${DEFAULT_PKG_TO_TEST} | grep -v "server/datastore/mysql/migrations" | grep -v "cmd/fleetctl" | grep -v "server/vulnerabilities" | sed -e 's|github.com/fleetdm/fleet/v4/||g') -else ifeq ($(CI_TEST_PKG), migration) - CI_PKG_TO_TEST="server/datastore/mysql/migrations/..." + CI_PKG_TO_TEST=$(shell go list ${DEFAULT_PKG_TO_TEST} | grep -v "server/datastore/mysql" | grep -v "cmd/fleetctl" | grep -v "server/vulnerabilities" | sed -e 's|github.com/fleetdm/fleet/v4/||g') +else ifeq ($(CI_TEST_PKG), mysql) + CI_PKG_TO_TEST="server/datastore/mysql/..." else ifeq ($(CI_TEST_PKG), fleetctl) CI_PKG_TO_TEST="cmd/fleetctl/..." else ifeq ($(CI_TEST_PKG), vuln) From b7c4f342930c8c1d08c3df6acd5aad04d3e6246c Mon Sep 17 00:00:00 2001 From: dantecatalfamo Date: Thu, 9 Jan 2025 09:38:20 -0500 Subject: [PATCH 6/6] Uncomment dependencies --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index fa8ec9dac69f..6ffdca38de79 100644 --- a/Makefile +++ b/Makefile @@ -201,7 +201,7 @@ debug-go-tests: @MYSQL_TEST=1 REDIS_TEST=1 MINIO_STORAGE_TEST=1 SAML_IDP_TEST=1 NETWORK_TEST=1 make .debug-go-tests # Command used in CI to run all tests. -test-go: # dump-test-schema generate-mock +test-go: dump-test-schema generate-mock make .run-go-tests PKG_TO_TEST="$(CI_PKG_TO_TEST)" analyze-go: