diff --git a/pkg/controllers/rolloutrun/executor/batch.go b/pkg/controllers/rolloutrun/executor/batch.go index 26a6e82..680e96c 100644 --- a/pkg/controllers/rolloutrun/executor/batch.go +++ b/pkg/controllers/rolloutrun/executor/batch.go @@ -20,6 +20,7 @@ import ( "fmt" "time" + corev1 "k8s.io/api/core/v1" utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/intstr" rolloutv1alpha1 "kusionstack.io/kube-api/rollout/v1alpha1" @@ -194,6 +195,8 @@ func (e *batchExecutor) doBatchUpgrading(ctx *ExecutorContext) (bool, time.Durat newStatus := ctx.NewStatus currentBatchIndex := newStatus.BatchStatus.CurrentBatchIndex currentBatch := rolloutRun.Spec.Batch.Batches[currentBatchIndex] + totalBatches := len(rolloutRun.Spec.Batch.Batches) + isLastBatch := int(currentBatchIndex+1) == totalBatches logger := ctx.GetBatchLogger() @@ -214,13 +217,15 @@ func (e *batchExecutor) doBatchUpgrading(ctx *ExecutorContext) (bool, time.Durat currentBatchExpectedReplicas, _ := workload.CalculateUpdatedReplicas(&status.Replicas, item.Replicas) - if info.CheckUpdatedReady(currentBatchExpectedReplicas) { + ready, reason := info.CheckUpdatedReady(currentBatchExpectedReplicas, isLastBatch) + if ready { // if the target is ready, we will not change partition continue } + ctx.Recorder.Eventf(ctx.RolloutRun, corev1.EventTypeNormal, "WaitingWorkloadUpdatedReady", "still waiting for target to be ready, target: %v, reason: %s", item.CrossClusterObjectNameReference, reason) allWorkloadReady = false - logger.V(3).Info("still waiting for target to be ready", "target", item.CrossClusterObjectNameReference) + logger.V(3).Info("still waiting for target to be ready", "target", item.CrossClusterObjectNameReference, "reason", reason) expectedReplicas, err := e.calculateExpectedReplicasBySlidingWindow(status, currentBatchExpectedReplicas, item.ReplicaSlidingWindow) if err != nil { diff --git a/pkg/controllers/rolloutrun/executor/canary.go b/pkg/controllers/rolloutrun/executor/canary.go index 4ec7849..f465c7d 100644 --- a/pkg/controllers/rolloutrun/executor/canary.go +++ b/pkg/controllers/rolloutrun/executor/canary.go @@ -224,7 +224,7 @@ func (e *canaryExecutor) doCanary(ctx *ExecutorContext) (bool, time.Duration, er // 2.b. waiting canary workload ready for _, info := range canaryWorkloads { - if !info.CheckUpdatedReady(info.Status.DesiredReplicas) { + if ready, _ := info.CheckUpdatedReady(info.Status.DesiredReplicas, false); !ready { // ready logger.Info("still waiting for canary target ready", "cluster", info.ClusterName, diff --git a/pkg/controllers/rolloutrun/webhook/worker.go b/pkg/controllers/rolloutrun/webhook/worker.go index 251216d..f2abf1b 100644 --- a/pkg/controllers/rolloutrun/webhook/worker.go +++ b/pkg/controllers/rolloutrun/webhook/worker.go @@ -235,7 +235,7 @@ func (w *worker) doProbe() (keepGoing bool) { func (w *worker) now() metav1.Time { now := metav1.NewTime(w.webhookManager.clock.Now()) data, _ := now.MarshalJSON() - now.UnmarshalJSON(data) + now.UnmarshalJSON(data) //nolint return now } diff --git a/pkg/workload/info.go b/pkg/workload/info.go index 4240eec..2b9074b 100644 --- a/pkg/workload/info.go +++ b/pkg/workload/info.go @@ -102,11 +102,17 @@ func (o *Info) String() string { return rolloutv1alpha1.CrossClusterObjectNameReference{Cluster: o.ClusterName, Name: o.Name}.String() } -func (o *Info) CheckUpdatedReady(replicas int32) bool { +func (o *Info) CheckUpdatedReady(replicas int32, strictCheck bool) (bool, string) { if o.Generation != o.Status.ObservedGeneration { - return false + return false, "workload Generation and ObservedGeneration are mismatched" } - return o.Status.UpdatedAvailableReplicas >= replicas && o.Status.ObservedReplicas <= o.Status.DesiredReplicas + if o.Status.UpdatedAvailableReplicas < replicas { + return false, "workload updated available replicas is not satisfied" + } + if strictCheck && o.Status.ObservedReplicas > o.Status.DesiredReplicas { + return false, "workload observed replicas is more than desiredReplicas" + } + return true, "" } func (o *Info) APIStatus() rolloutv1alpha1.RolloutWorkloadStatus {