Implement a pause between zone updates

Author: vrutkovs

api/operator/v1alpha1/vmdistributedcluster_types.go

@@ -36,6 +36,9 @@ type VMDistributedClusterSpec struct {
 	// ReadyDeadline is the deadline for the VMCluster to be ready.
 	// +optional
 	ReadyDeadline *metav1.Duration `json:"readyDeadline,omitempty"`
+	// ZoneUpdatePause is the time the operator should wait between zone updates to ensure a smooth transition.
+	// +optional
+	ZoneUpdatePause *metav1.Duration `json:"zoneUpdatePause,omitempty"`
 	// VMAgent points to the VMAgent object for collecting metrics from multiple VMClusters
 	VMAgent corev1.LocalObjectReference `json:"vmagent,omitempty"`
 	// VMUsers is a list of VMUser objects controlling traffic distribution between multiple VMClusters

api/operator/v1alpha1/zz_generated.deepcopy.go

@@ -15102,6 +15102,8 @@ spec:
                     type: object
                     x-kubernetes-map-type: atomic
                   type: array
+                zoneUpdatePause:
+                  type: string
                 zones:
                   properties:
                     vmclusters:

config/crd/overlay/crd.yaml

@@ -50,6 +50,7 @@ const (
 
 var (
 	defaultVMClusterWaitReadyDeadline = metav1.Duration{Duration: 5 * time.Minute}
+	defaultZoneUpdatePause            = metav1.Duration{Duration: 1 * time.Minute}
 )
 
 // CreateOrUpdate handles VM deployment reconciliation.
@@ -79,10 +80,15 @@ func CreateOrUpdate(ctx context.Context, cr *vmv1alpha1.VMDistributedCluster, rc
 		}
 	}
 
+	// Setup deadlines and timeouts
 	vmclusterWaitReadyDeadline := defaultVMClusterWaitReadyDeadline.Duration
 	if cr.Spec.ReadyDeadline != nil {
 		vmclusterWaitReadyDeadline = cr.Spec.ReadyDeadline.Duration
 	}
+	zoneUpdatePause := defaultZoneUpdatePause.Duration
+	if cr.Spec.ZoneUpdatePause != nil {
+		zoneUpdatePause = cr.Spec.ZoneUpdatePause.Duration
+	}
 
 	// Fetch global vmagent
 	vmAgentObj, err := fetchVMAgent(ctx, rclient, cr.Namespace, cr.Spec.VMAgent)
@@ -126,9 +132,6 @@ func CreateOrUpdate(ctx context.Context, cr *vmv1alpha1.VMDistributedCluster, rc
 		return fmt.Errorf("unexpected generations or zones config change detected: %v", diff)
 	}
 
-	// TODO[vrutkovs]: Mark all VMClusters as paused?
-	// This would prevent other reconciliation loops from changing them
-
 	// Disable VMClusters one by one if overrideSpec needs to be applied
 	httpClient := &http.Client{
 		Timeout: httpTimeout,
@@ -196,6 +199,9 @@ func CreateOrUpdate(ctx context.Context, cr *vmv1alpha1.VMDistributedCluster, rc
 		if err := setVMClusterStatusInVMUsers(ctx, rclient, cr, vmClusterObj, vmUserObjs, true); err != nil {
 			return fmt.Errorf("failed to set VMCluster status in VMUsers: %w", err)
 		}
+
+		// Sleep for zoneUpdatePause time between VMClusters updates
+		time.Sleep(zoneUpdatePause)
 	}
 	return nil
 }