Remove panic recovery in CI tests (#22644)

.github/workflows/test-go.yaml

@@ -120,6 +120,7 @@ jobs:
         fi
         GO_TEST_EXTRA_FLAGS="-v -race=$RACE_ENABLED -timeout=$GO_TEST_TIMEOUT $RUN_TESTS_ARG" \
           TEST_LOCK_FILE_PATH=$(pwd)/lock \
+          TEST_CRON_NO_RECOVER=1 \
           NETWORK_TEST=1 \
           REDIS_TEST=1 \
           MYSQL_TEST=1 \

cmd/fleet/serve_test.go

@@ -241,6 +241,14 @@ func TestAutomationsSchedule(t *testing.T) {
 	}))
 	defer ts.Close()
 
+	ds.TeamsSummaryFunc = func(ctx context.Context) ([]*fleet.TeamSummary, error) {
+		return []*fleet.TeamSummary{}, nil
+	}
+
+	ds.OutdatedAutomationBatchFunc = func(ctx context.Context) ([]fleet.PolicyFailure, error) {
+		return []fleet.PolicyFailure{}, nil
+	}
+
 	ds.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) {
 		return &fleet.AppConfig{
 			WebhookSettings: fleet.WebhookSettings{
@@ -620,6 +628,18 @@ func TestCronVulnerabilitiesSkipMkdirIfDisabled(t *testing.T) {
 		return nil
 	}
 
+	ds.ReconcileSoftwareTitlesFunc = func(ctx context.Context) error {
+		return nil
+	}
+
+	ds.SyncHostsSoftwareTitlesFunc = func(ctx context.Context, updatedAt time.Time) error {
+		return nil
+	}
+
+	ds.UpdateHostIssuesVulnerabilitiesFunc = func(ctx context.Context) error {
+		return nil
+	}
+
 	mockLocker := schedule.SetupMockLocker("vulnerabilities", "test_instance", time.Now().UTC())
 	ds.LockFunc = mockLocker.Lock
 	ds.UnlockFunc = mockLocker.Unlock
@@ -675,6 +695,15 @@ func TestAutomationsScheduleLockDuration(t *testing.T) {
 		}
 		return &ac, nil
 	}
+
+	ds.TeamsSummaryFunc = func(ctx context.Context) ([]*fleet.TeamSummary, error) {
+		return []*fleet.TeamSummary{}, nil
+	}
+
+	ds.OutdatedAutomationBatchFunc = func(ctx context.Context) ([]fleet.PolicyFailure, error) {
+		return []fleet.PolicyFailure{}, nil
+	}
+
 	hostStatus := make(chan struct{})
 	hostStatusClosed := false
 	failingPolicies := make(chan struct{})
@@ -739,6 +768,14 @@ func TestAutomationsScheduleIntervalChange(t *testing.T) {
 	}
 	configLoaded := make(chan struct{}, 1)
 
+	ds.TeamsSummaryFunc = func(ctx context.Context) ([]*fleet.TeamSummary, error) {
+		return []*fleet.TeamSummary{}, nil
+	}
+
+	ds.OutdatedAutomationBatchFunc = func(ctx context.Context) ([]fleet.PolicyFailure, error) {
+		return []fleet.PolicyFailure{}, nil
+	}
+
 	ds.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) {
 		select {
 		case configLoaded <- struct{}{}:

server/service/integration_mdm_test.go

@@ -8238,7 +8238,10 @@ func (s *integrationMDMTestSuite) runIntegrationsSchedule() {
 	// In testing, this can cause the test to hang until the next scheduled run. It isn't a very
 	// noticeable issue here since the intervals for these schedules are short.
 	ch := make(chan bool)
-	s.onIntegrationsScheduleDone = func() { close(ch) }
+	var once sync.Once
+	s.onIntegrationsScheduleDone = func() {
+		once.Do(func() { close(ch) })
+	}
 	_, err := s.integrationsSchedule.Trigger()
 	require.NoError(s.T(), err)
 	<-ch

server/service/schedule/schedule.go

@@ -7,6 +7,7 @@ package schedule
 import (
 	"context"
 	"fmt"
+	"os"
 	"sync"
 	"time"
 
@@ -469,8 +470,10 @@ func (s *Schedule) runAllJobs() {
 // runJob executes the job function with panic recovery.
 func runJob(ctx context.Context, fn JobFn) (err error) {
 	defer func() {
-		if r := recover(); r != nil {
-			err = fmt.Errorf("%v", r)
+		if os.Getenv("TEST_CRON_NO_RECOVER") != "1" { // for detecting panics in tests
+			if r := recover(); r != nil {
+				err = fmt.Errorf("%v", r)
+			}
 		}
 	}()
 

server/service/schedule/schedule_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"os"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -297,6 +298,8 @@ func TestConfigReloadCheck(t *testing.T) {
 }
 
 func TestJobPanicRecover(t *testing.T) {
+	os.Setenv("TEST_CRON_NO_RECOVER", "0")
+	defer os.Unsetenv("TEST_CRON_NO_RECOVER")
 	ctx, cancel := context.WithCancel(context.Background())
 
 	jobRan := false