From 1ecdad24ad82bc44856fde8142e858988d2cb98c Mon Sep 17 00:00:00 2001
From: Tim Lee <timlee@fleetdm.com>
Date: Wed, 9 Oct 2024 18:29:14 -0600
Subject: [PATCH] Remove panic recovery in CI tests (#22644)

---
 .github/workflows/test-go.yaml           |  1 +
 cmd/fleet/serve_test.go                  | 37 ++++++++++++++++++++++++
 server/service/integration_mdm_test.go   |  5 +++-
 server/service/schedule/schedule.go      |  7 +++--
 server/service/schedule/schedule_test.go |  3 ++
 5 files changed, 50 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test-go.yaml b/.github/workflows/test-go.yaml
index 10668436841a..b9c013bd4101 100644
--- a/.github/workflows/test-go.yaml
+++ b/.github/workflows/test-go.yaml
@@ -120,6 +120,7 @@ jobs:
         fi
         GO_TEST_EXTRA_FLAGS="-v -race=$RACE_ENABLED -timeout=$GO_TEST_TIMEOUT $RUN_TESTS_ARG" \
           TEST_LOCK_FILE_PATH=$(pwd)/lock \
+          TEST_CRON_NO_RECOVER=1 \
           NETWORK_TEST=1 \
           REDIS_TEST=1 \
           MYSQL_TEST=1 \
diff --git a/cmd/fleet/serve_test.go b/cmd/fleet/serve_test.go
index 0d147f698603..9763e3476d1b 100644
--- a/cmd/fleet/serve_test.go
+++ b/cmd/fleet/serve_test.go
@@ -241,6 +241,14 @@ func TestAutomationsSchedule(t *testing.T) {
 	}))
 	defer ts.Close()
 
+	ds.TeamsSummaryFunc = func(ctx context.Context) ([]*fleet.TeamSummary, error) {
+		return []*fleet.TeamSummary{}, nil
+	}
+
+	ds.OutdatedAutomationBatchFunc = func(ctx context.Context) ([]fleet.PolicyFailure, error) {
+		return []fleet.PolicyFailure{}, nil
+	}
+
 	ds.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) {
 		return &fleet.AppConfig{
 			WebhookSettings: fleet.WebhookSettings{
@@ -620,6 +628,18 @@ func TestCronVulnerabilitiesSkipMkdirIfDisabled(t *testing.T) {
 		return nil
 	}
 
+	ds.ReconcileSoftwareTitlesFunc = func(ctx context.Context) error {
+		return nil
+	}
+
+	ds.SyncHostsSoftwareTitlesFunc = func(ctx context.Context, updatedAt time.Time) error {
+		return nil
+	}
+
+	ds.UpdateHostIssuesVulnerabilitiesFunc = func(ctx context.Context) error {
+		return nil
+	}
+
 	mockLocker := schedule.SetupMockLocker("vulnerabilities", "test_instance", time.Now().UTC())
 	ds.LockFunc = mockLocker.Lock
 	ds.UnlockFunc = mockLocker.Unlock
@@ -675,6 +695,15 @@ func TestAutomationsScheduleLockDuration(t *testing.T) {
 		}
 		return &ac, nil
 	}
+
+	ds.TeamsSummaryFunc = func(ctx context.Context) ([]*fleet.TeamSummary, error) {
+		return []*fleet.TeamSummary{}, nil
+	}
+
+	ds.OutdatedAutomationBatchFunc = func(ctx context.Context) ([]fleet.PolicyFailure, error) {
+		return []fleet.PolicyFailure{}, nil
+	}
+
 	hostStatus := make(chan struct{})
 	hostStatusClosed := false
 	failingPolicies := make(chan struct{})
@@ -739,6 +768,14 @@ func TestAutomationsScheduleIntervalChange(t *testing.T) {
 	}
 	configLoaded := make(chan struct{}, 1)
 
+	ds.TeamsSummaryFunc = func(ctx context.Context) ([]*fleet.TeamSummary, error) {
+		return []*fleet.TeamSummary{}, nil
+	}
+
+	ds.OutdatedAutomationBatchFunc = func(ctx context.Context) ([]fleet.PolicyFailure, error) {
+		return []fleet.PolicyFailure{}, nil
+	}
+
 	ds.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) {
 		select {
 		case configLoaded <- struct{}{}:
diff --git a/server/service/integration_mdm_test.go b/server/service/integration_mdm_test.go
index 25e9664a7819..97ae14d8202f 100644
--- a/server/service/integration_mdm_test.go
+++ b/server/service/integration_mdm_test.go
@@ -8238,7 +8238,10 @@ func (s *integrationMDMTestSuite) runIntegrationsSchedule() {
 	// In testing, this can cause the test to hang until the next scheduled run. It isn't a very
 	// noticeable issue here since the intervals for these schedules are short.
 	ch := make(chan bool)
-	s.onIntegrationsScheduleDone = func() { close(ch) }
+	var once sync.Once
+	s.onIntegrationsScheduleDone = func() {
+		once.Do(func() { close(ch) })
+	}
 	_, err := s.integrationsSchedule.Trigger()
 	require.NoError(s.T(), err)
 	<-ch
diff --git a/server/service/schedule/schedule.go b/server/service/schedule/schedule.go
index 3e8f512987c0..11e8cbf9d8a5 100644
--- a/server/service/schedule/schedule.go
+++ b/server/service/schedule/schedule.go
@@ -7,6 +7,7 @@ package schedule
 import (
 	"context"
 	"fmt"
+	"os"
 	"sync"
 	"time"
 
@@ -469,8 +470,10 @@ func (s *Schedule) runAllJobs() {
 // runJob executes the job function with panic recovery.
 func runJob(ctx context.Context, fn JobFn) (err error) {
 	defer func() {
-		if r := recover(); r != nil {
-			err = fmt.Errorf("%v", r)
+		if os.Getenv("TEST_CRON_NO_RECOVER") != "1" { // for detecting panics in tests
+			if r := recover(); r != nil {
+				err = fmt.Errorf("%v", r)
+			}
 		}
 	}()
 
diff --git a/server/service/schedule/schedule_test.go b/server/service/schedule/schedule_test.go
index c54bd807fa88..0acda392fb77 100644
--- a/server/service/schedule/schedule_test.go
+++ b/server/service/schedule/schedule_test.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"os"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -297,6 +298,8 @@ func TestConfigReloadCheck(t *testing.T) {
 }
 
 func TestJobPanicRecover(t *testing.T) {
+	os.Setenv("TEST_CRON_NO_RECOVER", "0")
+	defer os.Unsetenv("TEST_CRON_NO_RECOVER")
 	ctx, cancel := context.WithCancel(context.Background())
 
 	jobRan := false