Debug syncer deadlock

.github/workflows/test.yml

@@ -49,25 +49,25 @@ jobs:
           host port: 3800
           mysql version: '8'
           mysql root password: test
-      - name: Test Stores
-        uses: n8maninger/action-golang-test@v1
-        with:
-          args: "-race;-short"
-      - name: Test Stores - MySQL
-        if: matrix.os == 'ubuntu-latest'
-        uses: n8maninger/action-golang-test@v1
-        env:
-          RENTERD_DB_URI: 127.0.0.1:3800
-          RENTERD_DB_USER: root
-          RENTERD_DB_PASSWORD: test
-        with:
-          package: "./stores"
-          args: "-race;-short"
+      # - name: Test Stores
+      #   uses: n8maninger/action-golang-test@v1
+      #   with:
+      #     args: "-race;-short"
+      # - name: Test Stores - MySQL
+      #   if: matrix.os == 'ubuntu-latest'
+      #   uses: n8maninger/action-golang-test@v1
+      #   env:
+      #     RENTERD_DB_URI: 127.0.0.1:3800
+      #     RENTERD_DB_USER: root
+      #     RENTERD_DB_PASSWORD: test
+      #   with:
+      #     package: "./stores"
+      #     args: "-race;-short"
       - name: Test Integration
         uses: n8maninger/action-golang-test@v1
         with:
           package: "./internal/test/e2e/..."
-          args: "-failfast;-race;-timeout=60m"
+          args: "-failfast;-race;-timeout=3h;-count=15"
       - name: Test Integration - MySQL
         if: matrix.os == 'ubuntu-latest'
         uses: n8maninger/action-golang-test@v1
@@ -77,6 +77,6 @@ jobs:
           RENTERD_DB_PASSWORD: test
         with:
           package: "./internal/test/e2e/..."
-          args: "-failfast;-race;-timeout=60m"
+          args: "-failfast;-race;-timeout=3h;-count=15"
       - name: Build
         run: go build -o bin/ ./cmd/renterd

cmd/renterd/node.go

@@ -389,7 +389,7 @@ func newBus(ctx context.Context, cfg config.Config, pk types.PrivateKey, network
 
 	return b, func(ctx context.Context) error {
 		return errors.Join(
-			s.Close(),
+			s.Shutdown(ctx),
 			w.Close(),
 			b.Shutdown(ctx),
 			sqlStore.Close(),

go.mod

@@ -14,7 +14,7 @@ require (
 	github.com/montanaflynn/stats v0.7.1
 	github.com/shopspring/decimal v1.4.0
 	go.sia.tech/core v0.4.6
-	go.sia.tech/coreutils v0.3.2
+	go.sia.tech/coreutils v0.3.3-0.20240911140040-2c7ca17f5292
 	go.sia.tech/gofakes3 v0.0.4
 	go.sia.tech/hostd v1.1.3-0.20240903081107-6e044db95238
 	go.sia.tech/jape v0.12.1

go.sum

@@ -74,8 +74,8 @@ go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0=
 go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I=
 go.sia.tech/core v0.4.6 h1:QLm97a7GWBonfnMEOokqWRAqsWCUPL7kzo6k3Adwx8E=
 go.sia.tech/core v0.4.6/go.mod h1:Zuq0Tn2aIXJyO0bjGu8cMeVWe+vwQnUfZhG1LCmjD5c=
-go.sia.tech/coreutils v0.3.2 h1:3gJqvs18n1FVZmcrnfIYyzS+rBu06OtIscDDAfUAYQI=
-go.sia.tech/coreutils v0.3.2/go.mod h1:woPVmN6GUpIKHdi71Hkb9goIbl7b45TquCsAyEzyxnI=
+go.sia.tech/coreutils v0.3.3-0.20240911140040-2c7ca17f5292 h1:zi0B4l71zhbUOhoGz9g8KLaf44lae99ZWtDeAANQohE=
+go.sia.tech/coreutils v0.3.3-0.20240911140040-2c7ca17f5292/go.mod h1:Y7L4NIozE9EyjrzY+e/vXog7o3wREu9e2POXFDsdnVE=
 go.sia.tech/gofakes3 v0.0.4 h1:Kvo8j5cVdJRBXvV1KBJ69bocY23twG8ao/HCdwuPMeI=
 go.sia.tech/gofakes3 v0.0.4/go.mod h1:6hh4lETCMbyFFNWp3FRE838geY6vh1Aeas7LtYDpQdc=
 go.sia.tech/hostd v1.1.3-0.20240903081107-6e044db95238 h1:DP9o+TnNeS34EmxZ/zqZ4px3DgL8en/2RL4EsiSd4GU=

internal/test/e2e/cluster.go

@@ -85,6 +85,8 @@ type TestCluster struct {
 	tt           test.TT
 	wk           types.PrivateKey
 	wg           sync.WaitGroup
+
+	tCancel context.CancelFunc
 }
 
 type dbConfig struct {
@@ -414,6 +416,16 @@ func newTestCluster(t *testing.T, opts testClusterOptions) *TestCluster {
 	autopilotShutdownFns = append(autopilotShutdownFns, autopilotServer.Shutdown)
 	autopilotShutdownFns = append(autopilotShutdownFns, ap.Shutdown)
 
+	tCtx, tCancel := context.WithCancel(context.Background())
+	go func() {
+		select {
+		case <-time.After(5 * time.Minute):
+			tt.Fatal("test is taking too long")
+		case <-tCtx.Done():
+			return
+		}
+	}()
+
 	network, genesis := testNetwork()
 	cluster := &TestCluster{
 		apID:         apCfg.ID,
@@ -438,6 +450,7 @@ func newTestCluster(t *testing.T, opts testClusterOptions) *TestCluster {
 		busShutdownFns:       busShutdownFns,
 		autopilotShutdownFns: autopilotShutdownFns,
 		s3ShutdownFns:        s3ShutdownFns,
+		tCancel:              tCancel,
 	}
 
 	// Spin up the servers.
@@ -638,7 +651,7 @@ func newTestBus(ctx context.Context, dir string, cfg config.Bus, cfgDb dbConfig,
 
 	shutdownFn := func(ctx context.Context) error {
 		return errors.Join(
-			s.Close(),
+			s.Shutdown(ctx),
 			w.Close(),
 			b.Shutdown(ctx),
 			sqlStore.Close(),
@@ -953,6 +966,7 @@ func (c *TestCluster) MineTransactions(ctx context.Context) error {
 // Shutdown shuts down a TestCluster.
 func (c *TestCluster) Shutdown() {
 	c.tt.Helper()
+	c.tCancel()
 	ctx := context.Background()
 	c.ShutdownAutopilot(ctx)
 	c.ShutdownS3(ctx)

internal/test/e2e/host.go

@@ -150,6 +150,9 @@ var defaultHostSettings = settings.Settings{
 
 // Close shutsdown the host
 func (h *Host) Close() error {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
+	defer cancel()
+
 	h.rhpv2.Close()
 	h.rhpv3.Close()
 	h.settings.Close()
@@ -159,7 +162,7 @@ func (h *Host) Close() error {
 	h.storage.Close()
 	h.store.Close()
 	h.syncerCancel()
-	h.s.Close()
+	h.s.Shutdown(ctx)
 	h.chainDB.Close()
 	return nil
 }