Maintain the previous state machine of share manager CR

1. Switch the share manager controller sync loop to sync volume first
   then sync share manager pod after
2. Skip calling gprc to mount the share manager volume if the share
   manager is currently delinquent. This helps to avoid the 20s delay when
   attemping to make the grpc call to dead share-manager pod

longhorn-6205

Signed-off-by: Phan Le <phan.le@suse.com>

controller/share_manager_controller.go

@@ -407,11 +407,11 @@ func (c *ShareManagerController) syncShareManager(key string) (err error) {
 		}
 	}()
 
-	if err = c.syncShareManagerPod(sm); err != nil {
+	if err = c.syncShareManagerVolume(sm); err != nil {
 		return err
 	}
 
-	if err = c.syncShareManagerVolume(sm); err != nil {
+	if err = c.syncShareManagerPod(sm); err != nil {
 		return err
 	}
 
@@ -638,6 +638,11 @@ func (c *ShareManagerController) syncShareManagerVolume(sm *longhorn.ShareManage
 		}
 	}()
 
+	isDelinquent, _, err := c.ds.IsRWXVolumeDelinquent(sm.Name)
+	if err != nil {
+		return err
+	}
+
 	if !c.isShareManagerRequiredForVolume(sm, volume, va) {
 		c.unmountShareManagerVolume(sm)
 
@@ -647,7 +652,7 @@ func (c *ShareManagerController) syncShareManagerVolume(sm *longhorn.ShareManage
 			sm.Status.State = longhorn.ShareManagerStateStopping
 		}
 		return nil
-	} else if sm.Status.State == longhorn.ShareManagerStateRunning {
+	} else if sm.Status.State == longhorn.ShareManagerStateRunning && !isDelinquent {
 		err := c.mountShareManagerVolume(sm)
 		if err != nil {
 			log.WithError(err).Error("Failed to mount share manager volume")