Merge pull request #593 from Icinga/wait-for-database-to-start-rather…

…-than-crashing-561 Merge network and database error retryability detection functions
Icinga · Jul 27, 2023 · 99de107 · 99de107
2 parents 5ea6831 + e776c99
commit 99de107
Show file tree

Hide file tree

Showing 3 changed files with 54 additions and 61 deletions.
diff --git a/pkg/icingadb/db.go b/pkg/icingadb/db.go
@@ -2,9 +2,7 @@ package icingadb
 
 import (
 	"context"
-	sqlDriver "database/sql/driver"
 	"fmt"
-	"github.com/go-sql-driver/mysql"
 	"github.com/icinga/icingadb/internal"
 	"github.com/icinga/icingadb/pkg/backoff"
 	"github.com/icinga/icingadb/pkg/com"
@@ -15,7 +13,6 @@ import (
 	"github.com/icinga/icingadb/pkg/retry"
 	"github.com/icinga/icingadb/pkg/utils"
 	"github.com/jmoiron/sqlx"
-	"github.com/lib/pq"
 	"github.com/pkg/errors"
 	"golang.org/x/sync/errgroup"
 	"golang.org/x/sync/semaphore"
@@ -339,7 +336,7 @@ func (db *DB) BulkExec(
 
 							return nil
 						},
-						IsRetryable,
+						retry.Retryable,
 						backoff.NewExponentialWithJitter(1*time.Millisecond, 1*time.Second),
 						retry.Settings{},
 					)
@@ -404,7 +401,7 @@ func (db *DB) NamedBulkExec(
 
 								return nil
 							},
-							IsRetryable,
+							retry.Retryable,
 							backoff.NewExponentialWithJitter(1*time.Millisecond, 1*time.Second),
 							retry.Settings{},
 						)
@@ -477,7 +474,7 @@ func (db *DB) NamedBulkExecTx(
 
 								return nil
 							},
-							IsRetryable,
+							retry.Retryable,
 							backoff.NewExponentialWithJitter(1*time.Millisecond, 1*time.Second),
 							retry.Settings{},
 						)
@@ -674,57 +671,3 @@ func (db *DB) log(ctx context.Context, query string, counter *com.Counter) perio
 		db.logger.Debugf("Finished executing %q with %d rows in %s", query, counter.Total(), tick.Elapsed)
 	}))
 }
-
-// IsRetryable checks whether the given error is retryable.
-func IsRetryable(err error) bool {
-	if errors.Is(err, sqlDriver.ErrBadConn) {
-		return true
-	}
-
-	if errors.Is(err, mysql.ErrInvalidConn) {
-		return true
-	}
-
-	var e *mysql.MySQLError
-	if errors.As(err, &e) {
-		switch e.Number {
-		case 1053, 1205, 1213, 2006:
-			// 1053: Server shutdown in progress
-			// 1205: Lock wait timeout
-			// 1213: Deadlock found when trying to get lock
-			// 2006: MySQL server has gone away
-			return true
-		default:
-			return false
-		}
-	}
-
-	var pe *pq.Error
-	if errors.As(err, &pe) {
-		switch pe.Code {
-		case "08000", // connection_exception
-			"08006", // connection_failure
-			"08001", // sqlclient_unable_to_establish_sqlconnection
-			"08004", // sqlserver_rejected_establishment_of_sqlconnection
-			"40001", // serialization_failure
-			"40P01", // deadlock_detected
-			"54000", // program_limit_exceeded
-			"55006", // object_in_use
-			"55P03", // lock_not_available
-			"57P01", // admin_shutdown
-			"57P02", // crash_shutdown
-			"57P03", // cannot_connect_now
-			"58000", // system_error
-			"58030", // io_error
-			"XX000": // internal_error
-			return true
-		default:
-			if strings.HasPrefix(string(pe.Code), "53") {
-				// Class 53 - Insufficient Resources
-				return true
-			}
-		}
-	}
-
-	return false
-}
diff --git a/pkg/icingadb/ha.go b/pkg/icingadb/ha.go
@@ -319,7 +319,7 @@ func (h *HA) realize(ctx context.Context, s *icingaredisv1.IcingaStatus, t *type
 
 			return nil
 		},
-		IsRetryable,
+		retry.Retryable,
 		backoff.NewExponentialWithJitter(time.Millisecond*256, time.Second*3),
 		retry.Settings{
 			OnError: func(_ time.Duration, attempt uint64, err, lastErr error) {

diff --git a/pkg/retry/retry.go b/pkg/retry/retry.go
@@ -2,9 +2,13 @@ package retry
 
 import (
 	"context"
+	"database/sql/driver"
+	"github.com/go-sql-driver/mysql"
 	"github.com/icinga/icingadb/pkg/backoff"
+	"github.com/lib/pq"
 	"github.com/pkg/errors"
 	"net"
+	"strings"
 	"syscall"
 	"time"
 )
@@ -130,5 +134,51 @@ func Retryable(err error) bool {
 		return true
 	}
 
+	if errors.Is(err, driver.ErrBadConn) {
+		return true
+	}
+	if errors.Is(err, mysql.ErrInvalidConn) {
+		return true
+	}
+
+	var e *mysql.MySQLError
+	if errors.As(err, &e) {
+		switch e.Number {
+		case 1053, 1205, 1213, 2006:
+			// 1053: Server shutdown in progress
+			// 1205: Lock wait timeout
+			// 1213: Deadlock found when trying to get lock
+			// 2006: MySQL server has gone away
+			return true
+		default:
+			return false
+		}
+	}
+
+	var pe *pq.Error
+	if errors.As(err, &pe) {
+		switch pe.Code {
+		case "08000", // connection_exception
+			"08006", // connection_failure
+			"08001", // sqlclient_unable_to_establish_sqlconnection
+			"08004", // sqlserver_rejected_establishment_of_sqlconnection
+			"40001", // serialization_failure
+			"40P01", // deadlock_detected
+			"54000", // program_limit_exceeded
+			"55006", // object_in_use
+			"55P03", // lock_not_available
+			"57P01", // admin_shutdown
+			"57P02", // crash_shutdown
+			"57P03", // cannot_connect_now
+			"58000", // system_error
+			"58030", // io_error
+			"XX000": // internal_error
+			return true
+		default:
+			// Class 53 - Insufficient Resources
+			return strings.HasPrefix(string(pe.Code), "53")
+		}
+	}
+
 	return false
 }