From 1f99e3c8a152a8a5a9addbbf8b8314314c4acbfe Mon Sep 17 00:00:00 2001 From: Simon_CQK Date: Thu, 28 Nov 2024 18:16:31 +0800 Subject: [PATCH] fix: sort broken mountpoint and put subpaths at the end. (#4416) Signed-off-by: SimonCqk Co-authored-by: SimonCqk --- pkg/csi/recover/recover.go | 14 +++++++---- pkg/utils/mountinfo/mountpoint.go | 41 +++++++++++++++++++++++++++---- 2 files changed, 45 insertions(+), 10 deletions(-) diff --git a/pkg/csi/recover/recover.go b/pkg/csi/recover/recover.go index 92ebfe541eb..a302c47b513 100644 --- a/pkg/csi/recover/recover.go +++ b/pkg/csi/recover/recover.go @@ -19,15 +19,11 @@ package recover import ( "context" "os" + "sort" "strconv" "strings" "time" - "github.com/fluid-cloudnative/fluid/pkg/common" - "github.com/fluid-cloudnative/fluid/pkg/utils" - "github.com/fluid-cloudnative/fluid/pkg/utils/dataset/volume" - "github.com/fluid-cloudnative/fluid/pkg/utils/kubelet" - "github.com/fluid-cloudnative/fluid/pkg/utils/mountinfo" "github.com/golang/glog" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" @@ -38,6 +34,12 @@ import ( "k8s.io/utils/mount" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" + + "github.com/fluid-cloudnative/fluid/pkg/common" + "github.com/fluid-cloudnative/fluid/pkg/utils" + "github.com/fluid-cloudnative/fluid/pkg/utils/dataset/volume" + "github.com/fluid-cloudnative/fluid/pkg/utils/kubelet" + "github.com/fluid-cloudnative/fluid/pkg/utils/mountinfo" ) const ( @@ -161,6 +163,8 @@ func (r *FuseRecover) recover() { return } + sort.Sort(brokenMounts) + for _, point := range brokenMounts { r.doRecover(point) } diff --git a/pkg/utils/mountinfo/mountpoint.go b/pkg/utils/mountinfo/mountpoint.go index 13f0edb2bd1..ae1b2f19906 100644 --- a/pkg/utils/mountinfo/mountpoint.go +++ b/pkg/utils/mountinfo/mountpoint.go @@ -21,8 +21,9 @@ import ( "path" "strings" - "github.com/fluid-cloudnative/fluid/pkg/utils" "github.com/golang/glog" + + "github.com/fluid-cloudnative/fluid/pkg/utils" ) type MountPoint struct { @@ -34,7 +35,7 @@ type MountPoint struct { NamespacedDatasetName string // - } -func GetBrokenMountPoints() ([]MountPoint, error) { +func GetBrokenMountPoints() (MountPoints, error) { // get mountinfo from proc mountByPath, err := loadMountInfo() if err != nil { @@ -81,6 +82,7 @@ func getBindMounts(mountByPath map[string]*Mount) (bindMountByName map[string][] bindMountByName = make(map[string][]*Mount) for k, m := range mountByPath { var datasetNamespacedName string + mCopy := m if strings.Contains(k, "kubernetes.io~csi") && strings.Contains(k, "mount") { // fluid bind mount target path is: /{kubeletRootDir}(default: /var/lib/kubelet)/pods/{podUID}/volumes/kubernetes.io~csi/{namespace}-{datasetName}/mount fields := strings.Split(k, "/") @@ -88,7 +90,7 @@ func getBindMounts(mountByPath map[string]*Mount) (bindMountByName map[string][] continue } datasetNamespacedName = fields[len(fields)-2] - bindMountByName[datasetNamespacedName] = append(bindMountByName[datasetNamespacedName], m) + bindMountByName[datasetNamespacedName] = append(bindMountByName[datasetNamespacedName], mCopy) } if strings.Contains(k, "volume-subpaths") { // pod using subPath, bind mount path is: /{kubeletRootDir}(default: /var/lib/kubelet)/pods/{podUID}/volume-subpaths/{namespace}-{datasetName}/{containerName}/{volumeIndex} @@ -97,7 +99,7 @@ func getBindMounts(mountByPath map[string]*Mount) (bindMountByName map[string][] continue } datasetNamespacedName = fields[len(fields)-3] - bindMountByName[datasetNamespacedName] = append(bindMountByName[datasetNamespacedName], m) + bindMountByName[datasetNamespacedName] = append(bindMountByName[datasetNamespacedName], mCopy) } } return @@ -111,7 +113,14 @@ func getBrokenBindMounts(globalMountByName map[string]*Mount, bindMountByName ma glog.V(6).Infof("ignoring mountpoint %s because of not finding its global mount point", name) continue } - for _, bindMount := range bindMounts { + for _, bm := range bindMounts { + bindMount := bm + + if strings.HasSuffix(bindMount.Subtree, "//deleted") { + glog.V(5).Infof("bindMount subtree has been deleted, trim /deleted suffix, bindmount: %v", bindMount) + bindMount.Subtree = strings.TrimSuffix(bindMount.Subtree, "//deleted") + } + // In case of not sharing same peer group in mount info, meaning it a broken mount point if len(utils.IntersectIntegerSets(bindMount.PeerGroups, globalMount.PeerGroups)) == 0 { brokenMounts = append(brokenMounts, MountPoint{ @@ -127,3 +136,25 @@ func getBrokenBindMounts(globalMountByName map[string]*Mount, bindMountByName ma } return } + +type MountPoints []MountPoint + +func (mp MountPoints) Len() int { return len(mp) } +func (mp MountPoints) Less(i, j int) bool { + if strings.Contains(mp[i].MountPath, "main") && !strings.Contains(mp[j].MountPath, "main") { + return false + } + if !strings.Contains(mp[i].MountPath, "main") && strings.Contains(mp[j].MountPath, "main") { + return true + } + if strings.Contains(mp[i].MountPath, "subpath") && !strings.Contains(mp[j].MountPath, "subpath") { + return false + } + if !strings.Contains(mp[i].MountPath, "subpath") && strings.Contains(mp[j].MountPath, "subpath") { + return true + } + return mp[i].MountPath < mp[j].MountPath +} +func (mp MountPoints) Swap(i, j int) { + mp[i].MountPath, mp[j].MountPath = mp[j].MountPath, mp[i].MountPath +}