fix(setting): do not delete the storage class

Delete the old storage class set in reconciling the
`default-longhorn-static-storage-class` setting.
Create the storage with a label if the storage class named as the
value of `default-longhorn-static-storage-class` setting does not
exist.

ref: longhorn/longhorn 9391

Signed-off-by: James Lu <james.lu@suse.com>

controller/setting_controller.go

@@ -139,7 +139,7 @@ func NewSettingController(
 	var err error
 	if _, err = ds.SettingInformer.AddEventHandlerWithResyncPeriod(cache.ResourceEventHandlerFuncs{
 		AddFunc:    sc.enqueueSetting,
-		UpdateFunc: func(old, cur interface{}) { sc.enqueueUpdateSetting(old, cur) },
+		UpdateFunc: func(old, cur interface{}) { sc.enqueueSetting(cur) },
 		DeleteFunc: sc.enqueueSetting,
 	}, settingControllerResyncPeriod); err != nil {
 		return nil, err
@@ -1078,31 +1078,53 @@ func (sc *SettingController) updateLogLevel(settingName types.SettingName) error
 }
 
 func (sc *SettingController) syncDefaultLonghornStaticStorageClass() error {
+	labels := types.GetDefaultLonghornStaticStorageClassLabels()
+
 	setting, err := sc.ds.GetSettingWithAutoFillingRO(types.SettingNameDefaultLonghornStaticStorageClass)
 	if err != nil {
 		return err
 	}
 
 	defaultStaticStorageClassName := setting.Value
 	_, err = sc.ds.GetStorageClassRO(defaultStaticStorageClassName)
-	if err != nil && apierrors.IsNotFound(err) {
-		allowVolumeExpansion := true
-		storageClass := &storagev1.StorageClass{
-			ObjectMeta: metav1.ObjectMeta{
-				Name: defaultStaticStorageClassName,
-			},
-			Provisioner:          types.LonghornDriverName,
-			AllowVolumeExpansion: &allowVolumeExpansion,
-			Parameters:           map[string]string{"staleReplicaTimeout": "30"},
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			allowVolumeExpansion := true
+			storageClass := &storagev1.StorageClass{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:   defaultStaticStorageClassName,
+					Labels: labels,
+				},
+				Provisioner:          types.LonghornDriverName,
+				AllowVolumeExpansion: &allowVolumeExpansion,
+				Parameters:           map[string]string{"staleReplicaTimeout": "30"},
+			}
+
+			if _, err := sc.ds.CreateStorageClass(storageClass); err != nil {
+				return err
+			}
+			return nil
 		}
+		return err
+	}
 
-		if _, err := sc.ds.CreateStorageClass(storageClass); err != nil {
-			return err
+	storageClassList, err := sc.ds.ListStorageClassesWithLabelsRO(labels)
+	if err != nil {
+		return err
+	}
+	for _, storageClass := range storageClassList {
+		if storageClass.Name == defaultStaticStorageClassName {
+			continue
+		}
+		if storageClass.DeletionTimestamp == nil {
+			sc.logger.Tracef("Deleting the old Longhorn static storage class %v", storageClass.Name)
+			if err := sc.ds.DeleteStorageClass(storageClass.Name); err != nil {
+				return err
+			}
 		}
-		return nil
 	}
 
-	return err
+	return nil
 }
 
 // updateDataEngine deletes the corresponding instance manager pods immediately if the data engine setting is disabled.
@@ -1444,31 +1466,6 @@ func (sc *SettingController) enqueueSetting(obj interface{}) {
 	sc.queue.Add(key)
 }
 
-func (sc *SettingController) enqueueUpdateSetting(oldObj, newObj interface{}) {
-	key, err := controller.KeyFunc(newObj)
-	if err != nil {
-		utilruntime.HandleError(fmt.Errorf("failed to get key for object %#v: %v", newObj, err))
-		return
-	}
-
-	oldSetting := oldObj.(*longhorn.Setting)
-	if oldSetting.Name == string(types.SettingNameDefaultLonghornStaticStorageClass) {
-		_, err := sc.ds.GetStorageClassRO(oldSetting.Value)
-		if err == nil {
-			if err := sc.ds.DeleteStorageClass(oldSetting.Value); err != nil {
-				utilruntime.HandleError(fmt.Errorf("failed to delete old %v for object %#v: %v", types.SettingNameDefaultLonghornStaticStorageClass, oldObj, err))
-				return
-			}
-		}
-		if err != nil && !apierrors.IsNotFound(err) {
-			utilruntime.HandleError(fmt.Errorf("failed to get old %v for object %#v: %v", types.SettingNameDefaultLonghornStaticStorageClass, oldObj, err))
-			return
-		}
-	}
-
-	sc.queue.Add(key)
-}
-
 func (sc *SettingController) enqueueSettingForNode(obj interface{}) {
 	if _, ok := obj.(*longhorn.Node); !ok {
 		// Ignore deleted node

datastore/kubernetes.go

@@ -390,6 +390,15 @@ func (s *DataStore) ListStorageClassesInPersistentVolumesWithLonghornProvisioner
 	return scList, nil
 }
 
+// ListStorageClassesWithLabelsRO returns a list of all StorageClasses with the given labels
+func (s *DataStore) ListStorageClassesWithLabelsRO(labels map[string]string) ([]*storagev1.StorageClass, error) {
+	selector, err := labelMapToLabelSelector(labels)
+	if err != nil {
+		return nil, err
+	}
+	return s.storageclassLister.List(selector)
+}
+
 // DeleteStorageClass deletes StorageClass with the given name
 func (s *DataStore) DeleteStorageClass(scName string) error {
 	return s.kubeClient.StorageV1().StorageClasses().Delete(context.TODO(), scName, metav1.DeleteOptions{})

types/types.go

@@ -176,6 +176,7 @@ const (
 	LonghornLabelVersion                    = "version"
 	LonghornLabelAdmissionWebhook           = "admission-webhook"
 	LonghornLabelConversionWebhook          = "conversion-webhook"
+	LonghornLabelManagedStorageClass        = "managed-storage-class"
 
 	LonghornRecoveryBackendServiceName = "longhorn-recovery-backend"
 
@@ -662,6 +663,16 @@ func GetVersionLabelKey() string {
 	return GetLonghornLabelKey(LonghornLabelVersion)
 }
 
+func GetManagedStorageClassLabelKey() string {
+	return GetLonghornLabelKey(LonghornLabelManagedStorageClass)
+}
+
+func GetDefaultLonghornStaticStorageClassLabels() map[string]string {
+	return map[string]string{
+		GetLonghornLabelKey(LonghornLabelManagedStorageClass): string(SettingNameDefaultLonghornStaticStorageClass),
+	}
+}
+
 func GetRegionAndZone(labels map[string]string) (string, string) {
 	region := ""
 	zone := ""