diff --git a/apis/apps/v1alpha1/well_known_labels.go b/apis/apps/v1alpha1/well_known_labels.go index 85e37780ef..85f42cb2d7 100644 --- a/apis/apps/v1alpha1/well_known_labels.go +++ b/apis/apps/v1alpha1/well_known_labels.go @@ -29,4 +29,7 @@ const ( // using in-place update strategy to kill sidecar. This image must be given if you want to use in-place update // strategy to terminate sidecar containers. KruiseTerminateSidecarWithImageEnv = "KRUISE_TERMINATE_SIDECAR_WHEN_JOB_EXIT_WITH_IMAGE" + + // KruiseIgnoreContainerExitCodeEnv is an env name, which represents a switch to ignore the exit code of sidecar container. + KruiseIgnoreContainerExitCodeEnv = "KRUISE_TERMINATE_SIDECAR_IGNORE_EXIT_CODE" ) diff --git a/pkg/controller/sidecarterminator/inplace_update_action.go b/pkg/controller/sidecarterminator/inplace_update_action.go index b684f3e30d..bb72d5a0b5 100644 --- a/pkg/controller/sidecarterminator/inplace_update_action.go +++ b/pkg/controller/sidecarterminator/inplace_update_action.go @@ -28,7 +28,7 @@ import ( "k8s.io/klog/v2" ) -func (r *ReconcileSidecarTerminator) executeInPlaceUpdateAction(originalPod *corev1.Pod, sidecars sets.String) error { +func (r *ReconcileSidecarTerminator) executeInPlaceUpdateAction(originalPod *corev1.Pod, sidecars sets.Set[string]) error { uncompletedSidecars := filterUncompletedSidecars(originalPod, sidecars) if uncompletedSidecars.Len() == 0 { return nil @@ -60,22 +60,26 @@ func (r *ReconcileSidecarTerminator) executeInPlaceUpdateAction(originalPod *cor klog.V(3).InfoS("SidecarTerminator -- InPlace update pod successfully", "pod", klog.KObj(originalPod)) r.recorder.Eventf(originalPod, corev1.EventTypeNormal, "SidecarTerminator", - "Kruise SidecarTerminator is trying to terminate sidecar %v using in-place update", uncompletedSidecars.List()) + "Kruise SidecarTerminator is trying to terminate sidecar %v using in-place update", uncompletedSidecars.UnsortedList()) } return err } // updateSidecarsForInPlaceUpdate replace sidecar image with KruiseTerminateSidecarWithImageEnv -func updateSidecarsForInPlaceUpdate(pod *corev1.Pod, sidecars sets.String) (bool, *corev1.Pod) { +func updateSidecarsForInPlaceUpdate(pod *corev1.Pod, sidecars sets.Set[string]) (bool, *corev1.Pod) { changed := false for i := range pod.Spec.Containers { container := &pod.Spec.Containers[i] if !sidecars.Has(container.Name) { continue } + newImage := getImageFromEnv(container) + if container.Image == newImage { + continue + } changed = true - container.Image = getImageFromEnv(container) + container.Image = newImage } return changed, pod } diff --git a/pkg/controller/sidecarterminator/kill_container_action.go b/pkg/controller/sidecarterminator/kill_container_action.go index cdf0356df5..bd13ef072f 100644 --- a/pkg/controller/sidecarterminator/kill_container_action.go +++ b/pkg/controller/sidecarterminator/kill_container_action.go @@ -29,7 +29,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -func (r *ReconcileSidecarTerminator) executeKillContainerAction(pod *corev1.Pod, sidecars sets.String) error { +func (r *ReconcileSidecarTerminator) executeKillContainerAction(pod *corev1.Pod, sidecars sets.Set[string]) error { uncompletedSidecars := filterUncompletedSidecars(pod, sidecars) if uncompletedSidecars.Len() == 0 { return nil @@ -47,7 +47,7 @@ func (r *ReconcileSidecarTerminator) executeKillContainerAction(pod *corev1.Pod, } var sidecarContainers []appsv1alpha1.ContainerRecreateRequestContainer - for _, name := range uncompletedSidecars.List() { + for _, name := range uncompletedSidecars.UnsortedList() { sidecarContainers = append(sidecarContainers, appsv1alpha1.ContainerRecreateRequestContainer{ Name: name, }) @@ -78,14 +78,14 @@ func (r *ReconcileSidecarTerminator) executeKillContainerAction(pod *corev1.Pod, klog.V(3).InfoS("SidecarTerminator -- Creating CRR successfully", "containerRecreateRequest", klog.KObj(crr)) r.recorder.Eventf(pod, corev1.EventTypeNormal, "SidecarTerminator", - "Kruise SidecarTerminator is trying to terminate sidecar %v using crr", uncompletedSidecars.List()) + "Kruise SidecarTerminator is trying to terminate sidecar %v using crr", uncompletedSidecars.UnsortedList()) } return err } -func filterUncompletedSidecars(pod *corev1.Pod, sidecars sets.String) sets.String { - uncompletedSidecars := sets.NewString(sidecars.List()...) +func filterUncompletedSidecars(pod *corev1.Pod, sidecars sets.Set[string]) sets.Set[string] { + uncompletedSidecars := sets.New[string](sidecars.UnsortedList()...) for i := range pod.Status.ContainerStatuses { status := &pod.Status.ContainerStatuses[i] if sidecars.Has(status.Name) && status.State.Terminated != nil { diff --git a/pkg/controller/sidecarterminator/sidecar_terminator_controller.go b/pkg/controller/sidecarterminator/sidecar_terminator_controller.go index 68ed90aeda..8232663c84 100644 --- a/pkg/controller/sidecarterminator/sidecar_terminator_controller.go +++ b/pkg/controller/sidecarterminator/sidecar_terminator_controller.go @@ -21,7 +21,6 @@ import ( "encoding/json" "flag" "fmt" - "strings" "time" corev1 "k8s.io/api/core/v1" @@ -38,7 +37,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" - appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" "github.com/openkruise/kruise/pkg/features" "github.com/openkruise/kruise/pkg/util" utilclient "github.com/openkruise/kruise/pkg/util/client" @@ -136,22 +134,25 @@ func (r *ReconcileSidecarTerminator) doReconcile(pod *corev1.Pod) (reconcile.Res if !isInterestingPod(pod) { return reconcile.Result{}, nil } - - sidecarNeedToExecuteKillContainer, sidecarNeedToExecuteInPlaceUpdate, err := r.groupSidecars(pod) - + vk, err := IsPodRunningOnVirtualKubelet(pod, r.Client) if err != nil { + klog.ErrorS(err, "SidecarTerminator -- Error occurred when try to check if pod is running on virtual-kubelet", "pod", klog.KObj(pod)) return reconcile.Result{}, err } - - if err := r.executeInPlaceUpdateAction(pod, sidecarNeedToExecuteInPlaceUpdate); err != nil { + normalSidecarNames, ignoreExitCodeSidecarNames, inplaceUpdateSidecarNames := getSidecarContainerNames(pod, vk) + if err = r.executeInPlaceUpdateAction(pod, inplaceUpdateSidecarNames); err != nil { return reconcile.Result{}, err } - + if err = r.executeKillContainerAction(pod, normalSidecarNames); err != nil { + return reconcile.Result{}, err + } + if ignoreExitCodeSidecarNames.Len() == 0 || !containersCompleted(pod, normalSidecarNames) || !containersCompleted(pod, inplaceUpdateSidecarNames) { + return reconcile.Result{}, nil + } if err := r.markJobPodTerminated(pod); err != nil { return reconcile.Result{}, err } - - if err := r.executeKillContainerAction(pod, sidecarNeedToExecuteKillContainer); err != nil { + if err := r.executeKillContainerAction(pod, ignoreExitCodeSidecarNames); err != nil { return reconcile.Result{}, err } @@ -184,8 +185,9 @@ func (r *ReconcileSidecarTerminator) markJobPodTerminated(pod *corev1.Pod) error }, } + mainContainers, _ := groupMainSidecarContainers(pod) // patch pod phase - if containersSucceeded(pod, getMain(pod)) { + if containersSucceeded(pod, mainContainers) { status.Phase = corev1.PodSucceeded } else { status.Phase = corev1.PodFailed @@ -203,32 +205,7 @@ func (r *ReconcileSidecarTerminator) markJobPodTerminated(pod *corev1.Pod) error return nil } -func (r *ReconcileSidecarTerminator) groupSidecars(pod *corev1.Pod) (sets.String, sets.String, error) { - runningOnVK, err := IsPodRunningOnVirtualKubelet(pod, r.Client) - if err != nil { - return nil, nil, client.IgnoreNotFound(err) - } - - inPlaceUpdate := sets.NewString() - killContainer := sets.NewString() - for i := range pod.Spec.Containers { - container := &pod.Spec.Containers[i] - for j := range container.Env { - if !runningOnVK && container.Env[j].Name == appsv1alpha1.KruiseTerminateSidecarEnv && - strings.EqualFold(container.Env[j].Value, "true") { - killContainer.Insert(container.Name) - break - } - if container.Env[j].Name == appsv1alpha1.KruiseTerminateSidecarWithImageEnv && - container.Env[j].Value != "" { - inPlaceUpdate.Insert(container.Name) - } - } - } - return killContainer, inPlaceUpdate, nil -} - -func containersCompleted(pod *corev1.Pod, containers sets.String) bool { +func containersCompleted(pod *corev1.Pod, containers sets.Set[string]) bool { if len(pod.Spec.Containers) != len(pod.Status.ContainerStatuses) { return false } @@ -242,7 +219,7 @@ func containersCompleted(pod *corev1.Pod, containers sets.String) bool { return true } -func containersSucceeded(pod *corev1.Pod, containers sets.String) bool { +func containersSucceeded(pod *corev1.Pod, containers sets.Set[string]) bool { if len(pod.Spec.Containers) != len(pod.Status.ContainerStatuses) { return false } diff --git a/pkg/controller/sidecarterminator/sidecar_terminator_controller_test.go b/pkg/controller/sidecarterminator/sidecar_terminator_controller_test.go index 99f42690fd..4347f1e17e 100644 --- a/pkg/controller/sidecarterminator/sidecar_terminator_controller_test.go +++ b/pkg/controller/sidecarterminator/sidecar_terminator_controller_test.go @@ -20,21 +20,20 @@ import ( "context" "encoding/json" "reflect" + "sort" "testing" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - + appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/reconcile" - - appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" ) const ( @@ -250,24 +249,6 @@ func TestKruiseDaemonStrategy(t *testing.T) { }, expectedPod: func() *corev1.Pod { pod := podDemo.DeepCopy() - pod.Status.Phase = corev1.PodFailed - return pod - }, - }, - { - name: "normal pod with sidecar, restartPolicy=Never, main containers failed and sidecar running", - getIn: func() *corev1.Pod { - podIn := podDemo.DeepCopy() - podIn.Status.ContainerStatuses[0] = failedMainContainerStatus - podIn.Status.ContainerStatuses[1] = runningSidecarContainerStatus - return podIn - }, - getCRR: func() *appsv1alpha1.ContainerRecreateRequest { - return crrDemo.DeepCopy() - }, - expectedPod: func() *corev1.Pod { - pod := podDemo.DeepCopy() - pod.Status.Phase = corev1.PodFailed return pod }, }, @@ -302,7 +283,6 @@ func TestKruiseDaemonStrategy(t *testing.T) { }, expectedPod: func() *corev1.Pod { pod := podDemo.DeepCopy() - pod.Status.Phase = corev1.PodSucceeded return pod }, }, @@ -387,7 +367,6 @@ func TestKruiseDaemonStrategy(t *testing.T) { }, expectedPod: func() *corev1.Pod { pod := podDemo.DeepCopy() - pod.Status.Phase = corev1.PodSucceeded return pod }, }, @@ -419,7 +398,6 @@ func TestKruiseDaemonStrategy(t *testing.T) { }, expectedPod: func() *corev1.Pod { pod := podDemo.DeepCopy() - pod.Status.Phase = corev1.PodSucceeded return pod }, }, @@ -451,7 +429,6 @@ func TestKruiseDaemonStrategy(t *testing.T) { }, expectedPod: func() *corev1.Pod { pod := podDemo.DeepCopy() - pod.Status.Phase = corev1.PodSucceeded return pod }, }, @@ -482,6 +459,82 @@ func TestKruiseDaemonStrategy(t *testing.T) { return pod }, }, + { + name: "normal pod with sidecar, restartPolicy=OnFailure, 2 succeeded main containers, 3 sidecars uncompleted", + getIn: func() *corev1.Pod { + podIn := podDemo.DeepCopy() + podIn.Spec.Containers = []corev1.Container{ + mainContainerFactory("main-1"), + mainContainerFactory("main-2"), + sidecarContainerFactory("sidecar-1", "true"), + sidecarContainerFactory("sidecar-2", "true"), + sidecarContainerFactory("sidecar-3", "true"), + } + podIn.Spec.Containers[4].Env = append(podIn.Spec.Containers[4].Env, corev1.EnvVar{ + Name: appsv1alpha1.KruiseIgnoreContainerExitCodeEnv, + Value: "true", + }) + podIn.Spec.RestartPolicy = corev1.RestartPolicyOnFailure + podIn.Status.ContainerStatuses = []corev1.ContainerStatus{ + rename(succeededMainContainerStatus.DeepCopy(), "main-1"), + rename(succeededMainContainerStatus.DeepCopy(), "main-2"), + rename(uncompletedSidecarContainerStatus.DeepCopy(), "sidecar-1"), + rename(uncompletedSidecarContainerStatus.DeepCopy(), "sidecar-2"), + rename(uncompletedSidecarContainerStatus.DeepCopy(), "sidecar-3"), + } + return podIn + }, + getCRR: func() *appsv1alpha1.ContainerRecreateRequest { + crr := crrDemo.DeepCopy() + crr.Spec.Containers = []appsv1alpha1.ContainerRecreateRequestContainer{ + {Name: "sidecar-2"}, + {Name: "sidecar-1"}, + } + return crr + }, + expectedPod: func() *corev1.Pod { + pod := podDemo.DeepCopy() + return pod + }, + }, + { + name: "normal pod with sidecar, restartPolicy=OnFailure, 2 succeeded main containers, 2 sidecars completed, 1 uncompleted", + getIn: func() *corev1.Pod { + podIn := podDemo.DeepCopy() + podIn.Spec.Containers = []corev1.Container{ + mainContainerFactory("main-1"), + mainContainerFactory("main-2"), + sidecarContainerFactory("sidecar-1", "true"), + sidecarContainerFactory("sidecar-2", "true"), + sidecarContainerFactory("sidecar-3", "true"), + } + podIn.Spec.Containers[4].Env = append(podIn.Spec.Containers[4].Env, corev1.EnvVar{ + Name: appsv1alpha1.KruiseIgnoreContainerExitCodeEnv, + Value: "true", + }) + podIn.Spec.RestartPolicy = corev1.RestartPolicyOnFailure + podIn.Status.ContainerStatuses = []corev1.ContainerStatus{ + rename(succeededMainContainerStatus.DeepCopy(), "main-1"), + rename(succeededMainContainerStatus.DeepCopy(), "main-2"), + rename(completedSidecarContainerStatus.DeepCopy(), "sidecar-1"), + rename(completedSidecarContainerStatus.DeepCopy(), "sidecar-2"), + rename(uncompletedSidecarContainerStatus.DeepCopy(), "sidecar-3"), + } + return podIn + }, + getCRR: func() *appsv1alpha1.ContainerRecreateRequest { + crr := crrDemo.DeepCopy() + crr.Spec.Containers = []appsv1alpha1.ContainerRecreateRequestContainer{ + {Name: "sidecar-3"}, + } + return crr + }, + expectedPod: func() *corev1.Pod { + pod := podDemo.DeepCopy() + pod.Status.Phase = corev1.PodSucceeded + return pod + }, + }, } for _, cs := range cases { @@ -514,9 +567,14 @@ func TestKruiseDaemonStrategy(t *testing.T) { realCRR.TypeMeta.APIVersion = appsv1alpha1.SchemeGroupVersion.String() realCRR.TypeMeta.Kind = "ContainerRecreateRequest" + if realCRR != nil { + sort.Sort(SortContainers(realCRR.Spec.Containers)) + } + if expectCRR != nil { + sort.Sort(SortContainers(expectCRR.Spec.Containers)) + } realBy, _ := json.Marshal(realCRR) expectBy, _ := json.Marshal(expectCRR) - if !(expectCRR == nil && errors.IsNotFound(err) || reflect.DeepEqual(realBy, expectBy)) { t.Fatal("Get unexpected CRR") } @@ -559,6 +617,7 @@ func TestInPlaceUpdateStrategy(t *testing.T) { podIn.Spec.Containers[1].Env = []corev1.EnvVar{ {Name: appsv1alpha1.KruiseTerminateSidecarWithImageEnv, Value: ExitQuicklyImage}, } + podIn.Spec.NodeName = vkNode.Name return podIn }, expectedNumber: 1, @@ -735,3 +794,15 @@ func rename(status *corev1.ContainerStatus, name string) corev1.ContainerStatus status.Name = name return *status } + +type SortContainers []appsv1alpha1.ContainerRecreateRequestContainer + +func (s SortContainers) Len() int { + return len(s) +} +func (s SortContainers) Swap(i, j int) { + s[i], s[j] = s[j], s[i] +} +func (s SortContainers) Less(i, j int) bool { + return s[i].Name < s[j].Name +} diff --git a/pkg/controller/sidecarterminator/sidecar_terminator_pod_event_handler.go b/pkg/controller/sidecarterminator/sidecar_terminator_pod_event_handler.go index 3b079ca73a..49bad035b3 100644 --- a/pkg/controller/sidecarterminator/sidecar_terminator_pod_event_handler.go +++ b/pkg/controller/sidecarterminator/sidecar_terminator_pod_event_handler.go @@ -18,18 +18,14 @@ package sidecarterminator import ( "context" - "strings" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/util/workqueue" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/reconcile" - - appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" ) var _ handler.TypedEventHandler[*corev1.Pod] = &enqueueRequestForPod{} @@ -75,55 +71,3 @@ func (p *enqueueRequestForPod) handlePodUpdate(q workqueue.RateLimitingInterface }) } } - -func isInterestingPod(pod *corev1.Pod) bool { - if pod.DeletionTimestamp != nil || - pod.Status.Phase == corev1.PodPending || - pod.Spec.RestartPolicy == corev1.RestartPolicyAlways { - return false - } - - sidecars := getSidecar(pod) - if sidecars.Len() == 0 || containersCompleted(pod, sidecars) { - return false - } - - switch pod.Spec.RestartPolicy { - case corev1.RestartPolicyNever: - return containersCompleted(pod, getMain(pod)) - case corev1.RestartPolicyOnFailure: - return containersSucceeded(pod, getMain(pod)) - } - return false -} - -func getMain(pod *corev1.Pod) sets.String { - mainNames := sets.NewString() - for i := range pod.Spec.Containers { - if !isSidecar(pod.Spec.Containers[i]) { - mainNames.Insert(pod.Spec.Containers[i].Name) - } - } - return mainNames -} - -func getSidecar(pod *corev1.Pod) sets.String { - sidecarNames := sets.NewString() - for i := range pod.Spec.Containers { - if isSidecar(pod.Spec.Containers[i]) { - sidecarNames.Insert(pod.Spec.Containers[i].Name) - } - } - return sidecarNames -} - -func isSidecar(container corev1.Container) bool { - for _, env := range container.Env { - if env.Name == appsv1alpha1.KruiseTerminateSidecarEnv && strings.EqualFold(env.Value, "true") { - return true - } else if env.Name == appsv1alpha1.KruiseTerminateSidecarWithImageEnv && env.Value != "" { - return true - } - } - return false -} diff --git a/pkg/controller/sidecarterminator/utils.go b/pkg/controller/sidecarterminator/utils.go new file mode 100644 index 0000000000..57973727d3 --- /dev/null +++ b/pkg/controller/sidecarterminator/utils.go @@ -0,0 +1,113 @@ +/* +Copyright 2023 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package sidecarterminator + +import ( + "strings" + + appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/sets" +) + +func isInterestingPod(pod *corev1.Pod) bool { + if pod.DeletionTimestamp != nil || + pod.Status.Phase == corev1.PodPending || + pod.Spec.RestartPolicy == corev1.RestartPolicyAlways { + return false + } + + mainContainers, sidecarContainers := groupMainSidecarContainers(pod) + if sidecarContainers.Len() == 0 || containersCompleted(pod, sidecarContainers) { + return false + } + + switch pod.Spec.RestartPolicy { + case corev1.RestartPolicyNever: + return containersCompleted(pod, mainContainers) + case corev1.RestartPolicyOnFailure: + return containersSucceeded(pod, mainContainers) + } + return false +} + +func groupMainSidecarContainers(pod *corev1.Pod) (sets.Set[string], sets.Set[string]) { + mainNames := sets.New[string]() + sidecarNames := sets.New[string]() + for i := range pod.Spec.Containers { + container := pod.Spec.Containers[i] + if isSidecarContainer(container) { + sidecarNames.Insert(pod.Spec.Containers[i].Name) + } else { + mainNames.Insert(pod.Spec.Containers[i].Name) + } + } + return mainNames, sidecarNames +} + +func getSidecarContainerNames(pod *corev1.Pod, vk bool) (sets.Set[string], sets.Set[string], sets.Set[string]) { + normalSidecarNames := sets.New[string]() + ignoreExitCodeSidecarNames := sets.New[string]() + inplaceUpdateSidecarNames := sets.New[string]() + + for i := range pod.Spec.Containers { + container := pod.Spec.Containers[i] + if vk && isInplaceUpdateSidecar(container) { + inplaceUpdateSidecarNames.Insert(container.Name) + } else if !vk && isIgnoreExitCodeSidecar(container) { + ignoreExitCodeSidecarNames.Insert(container.Name) + } else if !vk && isNormalSidecar(container) { + normalSidecarNames.Insert(container.Name) + } + } + + return normalSidecarNames, ignoreExitCodeSidecarNames, inplaceUpdateSidecarNames +} + +func isNormalSidecar(container corev1.Container) bool { + for _, env := range container.Env { + if env.Name == appsv1alpha1.KruiseTerminateSidecarEnv && strings.EqualFold(env.Value, "true") { + return true + } + } + return false +} + +func isIgnoreExitCodeSidecar(container corev1.Container) bool { + if !isNormalSidecar(container) { + return false + } + for _, env := range container.Env { + if env.Name == appsv1alpha1.KruiseIgnoreContainerExitCodeEnv && strings.EqualFold(env.Value, "true") { + return true + } + } + return false +} + +func isInplaceUpdateSidecar(container corev1.Container) bool { + for _, env := range container.Env { + if env.Name == appsv1alpha1.KruiseTerminateSidecarWithImageEnv && env.Value != "" { + return true + } + } + return false +} + +func isSidecarContainer(container corev1.Container) bool { + return isNormalSidecar(container) || isInplaceUpdateSidecar(container) +} diff --git a/pkg/controller/sidecarterminator/utils_test.go b/pkg/controller/sidecarterminator/utils_test.go new file mode 100644 index 0000000000..22d9c725f2 --- /dev/null +++ b/pkg/controller/sidecarterminator/utils_test.go @@ -0,0 +1,329 @@ +/* +Copyright 2023 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package sidecarterminator + +import ( + "testing" + + appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" +) + +func TestInterestingPod(t *testing.T) { + cases := []struct { + name string + getPod func() *corev1.Pod + expected bool + }{ + { + name: "Interesting, test1", + getPod: func() *corev1.Pod { + return &corev1.Pod{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "main", + }, + { + Name: "normal-sidecar1", + Env: []corev1.EnvVar{ + { + Name: appsv1alpha1.KruiseTerminateSidecarEnv, + Value: "true", + }, + }, + }, + }, + RestartPolicy: corev1.RestartPolicyNever, + }, + Status: corev1.PodStatus{ + ContainerStatuses: []corev1.ContainerStatus{ + { + Name: "main", + State: corev1.ContainerState{ + Terminated: &corev1.ContainerStateTerminated{ + ExitCode: 127, + }, + }, + }, + { + Name: "normal-sidecar1", + State: corev1.ContainerState{ + Running: &corev1.ContainerStateRunning{ + StartedAt: metav1.Now(), + }, + }, + }, + }, + Phase: corev1.PodRunning, + }, + } + }, + expected: true, + }, + { + name: "Interesting, test2", + getPod: func() *corev1.Pod { + return &corev1.Pod{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "main", + }, + { + Name: "normal-sidecar1", + Env: []corev1.EnvVar{ + { + Name: appsv1alpha1.KruiseTerminateSidecarEnv, + Value: "true", + }, + }, + }, + { + Name: "ignore-sidecar1", + Env: []corev1.EnvVar{ + { + Name: appsv1alpha1.KruiseTerminateSidecarEnv, + Value: "true", + }, + { + Name: appsv1alpha1.KruiseIgnoreContainerExitCodeEnv, + Value: "true", + }, + }, + }, + }, + RestartPolicy: corev1.RestartPolicyOnFailure, + }, + Status: corev1.PodStatus{ + ContainerStatuses: []corev1.ContainerStatus{ + { + Name: "main", + State: corev1.ContainerState{ + Terminated: &corev1.ContainerStateTerminated{ + ExitCode: 0, + }, + }, + }, + { + Name: "normal-sidecar1", + State: corev1.ContainerState{ + Terminated: &corev1.ContainerStateTerminated{ + ExitCode: 0, + }, + }, + }, + { + Name: "ignore-sidecar1", + State: corev1.ContainerState{ + Running: &corev1.ContainerStateRunning{ + StartedAt: metav1.Now(), + }, + }, + }, + }, + }, + } + }, + expected: true, + }, + { + name: "Interesting, test3", + getPod: func() *corev1.Pod { + return &corev1.Pod{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "main", + }, + { + Name: "normal-sidecar1", + Env: []corev1.EnvVar{ + { + Name: appsv1alpha1.KruiseTerminateSidecarEnv, + Value: "true", + }, + }, + }, + { + Name: "inplace-update-sidecar1", + Env: []corev1.EnvVar{ + { + Name: appsv1alpha1.KruiseTerminateSidecarWithImageEnv, + Value: "inplace-update/image", + }, + }, + }, + }, + RestartPolicy: corev1.RestartPolicyOnFailure, + }, + Status: corev1.PodStatus{ + ContainerStatuses: []corev1.ContainerStatus{ + { + Name: "main", + State: corev1.ContainerState{ + Terminated: &corev1.ContainerStateTerminated{ + ExitCode: 0, + }, + }, + }, + { + Name: "normal-sidecar1", + State: corev1.ContainerState{ + Terminated: &corev1.ContainerStateTerminated{ + ExitCode: 0, + }, + }, + }, + { + Name: "inplace-update-sidecar1", + State: corev1.ContainerState{ + Running: &corev1.ContainerStateRunning{ + StartedAt: metav1.Now(), + }, + }, + }, + }, + }, + } + }, + expected: true, + }, + } + + for _, cs := range cases { + pod := cs.getPod() + if isInterestingPod(pod) != cs.expected { + t.Errorf("case %s failed, expected %v, got %v", cs.name, cs.expected, isInterestingPod(pod)) + } + } +} + +func TestGetSidecarContainerNames(t *testing.T) { + cases := []struct { + name string + getPod func() *corev1.Pod + isVK bool + expected func() (sets.Set[string], sets.Set[string], sets.Set[string]) + }{ + { + name: "normal, test1", + isVK: false, + getPod: func() *corev1.Pod { + return &corev1.Pod{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "main", + }, + { + Name: "normal-sidecar1", + Env: []corev1.EnvVar{ + { + Name: appsv1alpha1.KruiseTerminateSidecarEnv, + Value: "true", + }, + { + Name: appsv1alpha1.KruiseTerminateSidecarWithImageEnv, + Value: "true", + }, + }, + }, + { + Name: "ignore-sidecar1", + Env: []corev1.EnvVar{ + { + Name: appsv1alpha1.KruiseTerminateSidecarEnv, + Value: "true", + }, + { + Name: appsv1alpha1.KruiseIgnoreContainerExitCodeEnv, + Value: "true", + }, + }, + }, + }, + }, + } + }, + expected: func() (sets.Set[string], sets.Set[string], sets.Set[string]) { + return sets.New[string]("normal-sidecar1"), sets.New[string]("ignore-sidecar1"), sets.New[string]() + }, + }, + { + name: "vk, test2", + isVK: true, + getPod: func() *corev1.Pod { + return &corev1.Pod{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "main", + }, + { + Name: "vk-sidecar1", + Env: []corev1.EnvVar{ + { + Name: appsv1alpha1.KruiseTerminateSidecarEnv, + Value: "true", + }, + { + Name: appsv1alpha1.KruiseTerminateSidecarWithImageEnv, + Value: "true", + }, + }, + }, + { + Name: "ignore-sidecar1", + Env: []corev1.EnvVar{ + { + Name: appsv1alpha1.KruiseTerminateSidecarEnv, + Value: "true", + }, + { + Name: appsv1alpha1.KruiseIgnoreContainerExitCodeEnv, + Value: "true", + }, + }, + }, + }, + }, + } + }, + expected: func() (sets.Set[string], sets.Set[string], sets.Set[string]) { + return sets.New[string](), sets.New[string](), sets.New[string]("vk-sidecar1") + }, + }, + } + + for _, cs := range cases { + pod := cs.getPod() + normalSidecarNames, ignoreExitCodeSidecarNames, inplaceUpdateSidecarNames := getSidecarContainerNames(pod, cs.isVK) + expected1, expected2, expected3 := cs.expected() + if !normalSidecarNames.Equal(expected1) { + t.Errorf("case %s failed, expected %v, got %v", cs.name, expected1, normalSidecarNames) + } + if !ignoreExitCodeSidecarNames.Equal(expected2) { + t.Errorf("case %s failed, expected %v, got %v", cs.name, expected2, ignoreExitCodeSidecarNames) + } + if !inplaceUpdateSidecarNames.Equal(expected3) { + t.Errorf("case %s failed, expected %v, got %v", cs.name, expected3, inplaceUpdateSidecarNames) + } + } +} diff --git a/test/e2e/apps/sidecarterminator.go b/test/e2e/apps/sidecarterminator.go index 0714a4c076..ec26439de4 100644 --- a/test/e2e/apps/sidecarterminator.go +++ b/test/e2e/apps/sidecarterminator.go @@ -62,6 +62,10 @@ var _ = SIGDescribe("SidecarTerminator", func() { Name: appsv1alpha1.KruiseTerminateSidecarEnv, Value: "true", }, + { + Name: appsv1alpha1.KruiseIgnoreContainerExitCodeEnv, + Value: "true", + }, }, }