fix: sort broken mountpoint and put subpaths at the end. (#4416)

Signed-off-by: SimonCqk <cqk0100@gamil.com>
Co-authored-by: SimonCqk <cqk0100@gamil.com>

pkg/csi/recover/recover.go

@@ -19,15 +19,11 @@ package recover
 import (
 	"context"
 	"os"
+	"sort"
 	"strconv"
 	"strings"
 	"time"
 
-	"github.com/fluid-cloudnative/fluid/pkg/common"
-	"github.com/fluid-cloudnative/fluid/pkg/utils"
-	"github.com/fluid-cloudnative/fluid/pkg/utils/dataset/volume"
-	"github.com/fluid-cloudnative/fluid/pkg/utils/kubelet"
-	"github.com/fluid-cloudnative/fluid/pkg/utils/mountinfo"
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
@@ -38,6 +34,12 @@ import (
 	"k8s.io/utils/mount"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
+
+	"github.com/fluid-cloudnative/fluid/pkg/common"
+	"github.com/fluid-cloudnative/fluid/pkg/utils"
+	"github.com/fluid-cloudnative/fluid/pkg/utils/dataset/volume"
+	"github.com/fluid-cloudnative/fluid/pkg/utils/kubelet"
+	"github.com/fluid-cloudnative/fluid/pkg/utils/mountinfo"
 )
 
 const (
@@ -161,6 +163,8 @@ func (r *FuseRecover) recover() {
 		return
 	}
 
+	sort.Sort(brokenMounts)
+
 	for _, point := range brokenMounts {
 		r.doRecover(point)
 	}

pkg/utils/mountinfo/mountpoint.go

@@ -21,8 +21,9 @@ import (
 	"path"
 	"strings"
 
-	"github.com/fluid-cloudnative/fluid/pkg/utils"
 	"github.com/golang/glog"
+
+	"github.com/fluid-cloudnative/fluid/pkg/utils"
 )
 
 type MountPoint struct {
@@ -34,7 +35,7 @@ type MountPoint struct {
 	NamespacedDatasetName string // <namespace>-<dataset>
 }
 
-func GetBrokenMountPoints() ([]MountPoint, error) {
+func GetBrokenMountPoints() (MountPoints, error) {
 	// get mountinfo from proc
 	mountByPath, err := loadMountInfo()
 	if err != nil {
@@ -81,14 +82,15 @@ func getBindMounts(mountByPath map[string]*Mount) (bindMountByName map[string][]
 	bindMountByName = make(map[string][]*Mount)
 	for k, m := range mountByPath {
 		var datasetNamespacedName string
+		mCopy := m
 		if strings.Contains(k, "kubernetes.io~csi") && strings.Contains(k, "mount") {
 			// fluid bind mount target path is: /{kubeletRootDir}(default: /var/lib/kubelet)/pods/{podUID}/volumes/kubernetes.io~csi/{namespace}-{datasetName}/mount
 			fields := strings.Split(k, "/")
 			if len(fields) < 3 {
 				continue
 			}
 			datasetNamespacedName = fields[len(fields)-2]
-			bindMountByName[datasetNamespacedName] = append(bindMountByName[datasetNamespacedName], m)
+			bindMountByName[datasetNamespacedName] = append(bindMountByName[datasetNamespacedName], mCopy)
 		}
 		if strings.Contains(k, "volume-subpaths") {
 			// pod using subPath, bind mount path is: /{kubeletRootDir}(default: /var/lib/kubelet)/pods/{podUID}/volume-subpaths/{namespace}-{datasetName}/{containerName}/{volumeIndex}
@@ -97,7 +99,7 @@ func getBindMounts(mountByPath map[string]*Mount) (bindMountByName map[string][]
 				continue
 			}
 			datasetNamespacedName = fields[len(fields)-3]
-			bindMountByName[datasetNamespacedName] = append(bindMountByName[datasetNamespacedName], m)
+			bindMountByName[datasetNamespacedName] = append(bindMountByName[datasetNamespacedName], mCopy)
 		}
 	}
 	return
@@ -111,7 +113,14 @@ func getBrokenBindMounts(globalMountByName map[string]*Mount, bindMountByName ma
 			glog.V(6).Infof("ignoring mountpoint %s because of not finding its global mount point", name)
 			continue
 		}
-		for _, bindMount := range bindMounts {
+		for _, bm := range bindMounts {
+			bindMount := bm
+
+			if strings.HasSuffix(bindMount.Subtree, "//deleted") {
+				glog.V(5).Infof("bindMount subtree has been deleted, trim /deleted suffix, bindmount: %v", bindMount)
+				bindMount.Subtree = strings.TrimSuffix(bindMount.Subtree, "//deleted")
+			}
+
 			// In case of not sharing same peer group in mount info, meaning it a broken mount point
 			if len(utils.IntersectIntegerSets(bindMount.PeerGroups, globalMount.PeerGroups)) == 0 {
 				brokenMounts = append(brokenMounts, MountPoint{
@@ -127,3 +136,25 @@ func getBrokenBindMounts(globalMountByName map[string]*Mount, bindMountByName ma
 	}
 	return
 }
+
+type MountPoints []MountPoint
+
+func (mp MountPoints) Len() int { return len(mp) }
+func (mp MountPoints) Less(i, j int) bool {
+	if strings.Contains(mp[i].MountPath, "main") && !strings.Contains(mp[j].MountPath, "main") {
+		return false
+	}
+	if !strings.Contains(mp[i].MountPath, "main") && strings.Contains(mp[j].MountPath, "main") {
+		return true
+	}
+	if strings.Contains(mp[i].MountPath, "subpath") && !strings.Contains(mp[j].MountPath, "subpath") {
+		return false
+	}
+	if !strings.Contains(mp[i].MountPath, "subpath") && strings.Contains(mp[j].MountPath, "subpath") {
+		return true
+	}
+	return mp[i].MountPath < mp[j].MountPath
+}
+func (mp MountPoints) Swap(i, j int) {
+	mp[i].MountPath, mp[j].MountPath = mp[j].MountPath, mp[i].MountPath
+}