diff --git a/clustering/operations.go b/clustering/operations.go index 4c149ced..0c5380c6 100644 --- a/clustering/operations.go +++ b/clustering/operations.go @@ -149,14 +149,31 @@ func (p *managerProcess) clone(ctx context.Context, ss *StatusSet) (bool, error) func (p *managerProcess) switchover(ctx context.Context, ss *StatusSet) error { log := logFromContext(ctx) log.Info("begin switchover the primary", "current", ss.Primary, "next", ss.Candidate) - pdb := ss.DBOps[ss.Primary] - if err := pdb.SetReadOnly(ctx, true); err != nil { - return fmt.Errorf("failed to make instance %d read-only: %w", ss.Primary, err) + readOnly := false + for i := 0; i < 2; i++ { + ctxTimeout, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + errCh := make(chan error) + go func() { + errCh <- pdb.SetReadOnly(ctxTimeout, true) + }() + err := <-errCh + if err != nil { + log.Error(err, "failed to set read-only mode", "instance", ss.Primary) + } else { + readOnly = true + } + time.Sleep(100 * time.Millisecond) + if err = pdb.KillConnections(ctx); err != nil { + return fmt.Errorf("failed to kill connections in instance %d: %w", ss.Primary, err) + } + if readOnly { + break + } } - time.Sleep(100 * time.Millisecond) - if err := pdb.KillConnections(ctx); err != nil { - return fmt.Errorf("failed to kill connections in instance %d: %w", ss.Primary, err) + if !readOnly { + return fmt.Errorf("failed to set read-only mode in instance %d", ss.Primary) } pst, err := pdb.GetStatus(ctx) if err != nil {