diff --git a/PROJECT b/PROJECT index 3303d58..531530e 100644 --- a/PROJECT +++ b/PROJECT @@ -98,4 +98,13 @@ resources: kind: ConsumeNamespaceResources path: stackzoo.io/khaos/api/v1alpha1 version: v1alpha1 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: stackzoo.io + group: khaos + kind: CordonNode + path: stackzoo.io/khaos/api/v1alpha1 + version: v1alpha1 version: "3" diff --git a/README.md b/README.md index 6f4ad59..af8d100 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ Currently, Khaos does not implement *cronjobs*; any scheduling of Khaos Custom R - [x] Delete cluster nodes - [X] Delete secrets - [X] Delete configmaps +- [X] Cordon nodes - [X] Inject resource constraints in pods - [X] Add o remove labels in pods - [X] Flood api server with calls @@ -87,17 +88,17 @@ Install and list all the available operator's CRDs with the following command: make install && kubectl get crds NAME CREATED AT -apiserveroverloads.khaos.stackzoo.io 2023-11-30T12:51:02Z -commandinjections.khaos.stackzoo.io 2023-11-30T12:51:02Z -configmapdestroyers.khaos.stackzoo.io 2023-11-30T12:51:02Z -consumenamespaceresources.khaos.stackzoo.io 2023-11-30T13:58:09Z -containerresourcechaos.khaos.stackzoo.io 2023-11-30T12:51:02Z -eventsentropies.khaos.stackzoo.io 2023-11-30T12:51:02Z -nodedestroyers.khaos.stackzoo.io 2023-11-30T12:51:02Z -poddestroyers.khaos.stackzoo.io 2023-11-30T12:51:02Z -podlabelchaos.khaos.stackzoo.io 2023-11-30T12:51:02Z -secretdestroyers.khaos.stackzoo.io 2023-11-30T12:51:02Z -serviceaccountremovers.khaos.stackzoo.io 2023-11-30T12:51:02Z +apiserveroverloads.khaos.stackzoo.io 2023-12-06T13:20:49Z +commandinjections.khaos.stackzoo.io 2023-12-06T13:20:49Z +configmapdestroyers.khaos.stackzoo.io 2023-12-06T13:20:49Z +consumenamespaceresources.khaos.stackzoo.io 2023-12-06T13:20:49Z +containerresourcechaos.khaos.stackzoo.io 2023-12-06T13:20:49Z +cordonnodes.khaos.stackzoo.io 2023-12-06T13:20:49Z +eventsentropies.khaos.stackzoo.io 2023-12-06T13:20:49Z +nodedestroyers.khaos.stackzoo.io 2023-12-06T13:20:49Z +poddestroyers.khaos.stackzoo.io 2023-12-06T13:20:49Z +podlabelchaos.khaos.stackzoo.io 2023-12-06T13:20:49Z +secretdestroyers.khaos.stackzoo.io 2023-12-06T13:20:49Z ``` In order to run the operator on your cluster (current context - i.e. whatever cluster `kubectl cluster-info` shows) run: @@ -616,11 +617,52 @@ kubectl get events | grep gibberish +
+ CORDON NODES +Apply the following `CordonNodes` manifest: +```yaml +apiVersion: khaos.stackzoo.io/v1alpha1 +kind: CordonNode +metadata: + name: example-cordon-node +spec: + nodesToCordon: + - test-operator-cluster-worker + - test-operator-cluster-worker2 + - test-operator-cluster-worker3 +``` + +```console +kubectl apply -f examples/cordon-nodes.yaml +``` + +Now check the status of the resource: +```console +kubectl describe cordonnodes.khaos.stackzoo.io example-cordon-node | grep "Nodes Cordoned" +Nodes Cordoned: 3 +``` + + +Now run a busybox pod: +```console +kubectl apply -f examples/test-node-cordon-pod.yaml + +pod/busybox-pod created +``` + +Let's check that pod: +```console +kubectl -n default describe pod busybox-pod | grep Warning + +Warning FailedScheduling 63s default-scheduler 0/4 nodes are available: 1 node(s) had untolerated taint {node-role.kubernetes.io/control-plane: }, 3 node(s) were unschedulable. preemption: 0/4 nodes are available: 4 Preemption is not helpful for scheduling.. +``` + +

diff --git a/api/v1alpha1/cordonnode_types.go b/api/v1alpha1/cordonnode_types.go new file mode 100644 index 0000000..4c83116 --- /dev/null +++ b/api/v1alpha1/cordonnode_types.go @@ -0,0 +1,58 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// CordonNodeSpec defines the desired state of CordonNode +type CordonNodeSpec struct { + // NodesToCordon is a list of node names to cordon + NodesToCordon []string `json:"nodesToCordon,omitempty"` +} + +// CordonNodeStatus defines the observed state of CordonNode +type CordonNodeStatus struct { + // NodesCordoned is the number of nodes successfully cordoned + NodesCordoned int `json:"nodesCordoned,omitempty"` +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status + +// CordonNode is the Schema for the cordonnodes API +type CordonNode struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec CordonNodeSpec `json:"spec,omitempty"` + Status CordonNodeStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// CordonNodeList contains a list of CordonNode +type CordonNodeList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []CordonNode `json:"items"` +} + +func init() { + SchemeBuilder.Register(&CordonNode{}, &CordonNodeList{}) +} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index ac214bb..6ee425d 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -474,6 +474,95 @@ func (in *ContainerResourceChaosStatus) DeepCopy() *ContainerResourceChaosStatus return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CordonNode) DeepCopyInto(out *CordonNode) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CordonNode. +func (in *CordonNode) DeepCopy() *CordonNode { + if in == nil { + return nil + } + out := new(CordonNode) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CordonNode) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CordonNodeList) DeepCopyInto(out *CordonNodeList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]CordonNode, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CordonNodeList. +func (in *CordonNodeList) DeepCopy() *CordonNodeList { + if in == nil { + return nil + } + out := new(CordonNodeList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CordonNodeList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CordonNodeSpec) DeepCopyInto(out *CordonNodeSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CordonNodeSpec. +func (in *CordonNodeSpec) DeepCopy() *CordonNodeSpec { + if in == nil { + return nil + } + out := new(CordonNodeSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CordonNodeStatus) DeepCopyInto(out *CordonNodeStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CordonNodeStatus. +func (in *CordonNodeStatus) DeepCopy() *CordonNodeStatus { + if in == nil { + return nil + } + out := new(CordonNodeStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *EventsEntropy) DeepCopyInto(out *EventsEntropy) { *out = *in diff --git a/cmd/main.go b/cmd/main.go index 20ea119..91e9e22 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -159,6 +159,13 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "ConsumeNamespaceResources") os.Exit(1) } + if err = (&controller.CordonNodeReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "CordonNode") + os.Exit(1) + } //+kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { diff --git a/config/crd/bases/khaos.stackzoo.io_cordonnodes.yaml b/config/crd/bases/khaos.stackzoo.io_cordonnodes.yaml new file mode 100644 index 0000000..9c665d4 --- /dev/null +++ b/config/crd/bases/khaos.stackzoo.io_cordonnodes.yaml @@ -0,0 +1,54 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.13.0 + name: cordonnodes.khaos.stackzoo.io +spec: + group: khaos.stackzoo.io + names: + kind: CordonNode + listKind: CordonNodeList + plural: cordonnodes + singular: cordonnode + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: CordonNode is the Schema for the cordonnodes API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: CordonNodeSpec defines the desired state of CordonNode + properties: + nodesToCordon: + description: NodesToCordon is a list of node names to cordon + items: + type: string + type: array + type: object + status: + description: CordonNodeStatus defines the observed state of CordonNode + properties: + nodesCordoned: + description: NodesCordoned is the number of nodes successfully cordoned + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 7ad0fe4..0a5a579 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -12,6 +12,7 @@ resources: - bases/khaos.stackzoo.io_apiserveroverloads.yaml - bases/khaos.stackzoo.io_eventsentropies.yaml - bases/khaos.stackzoo.io_consumenamespaceresources.yaml +- bases/khaos.stackzoo.io_cordonnodes.yaml #+kubebuilder:scaffold:crdkustomizeresource patches: @@ -27,6 +28,7 @@ patches: #- path: patches/webhook_in_apiserveroverloads.yaml #- path: patches/webhook_in_eventsentropies.yaml #- path: patches/webhook_in_consumenamespaceresources.yaml +#- path: patches/webhook_in_cordonnodes.yaml #+kubebuilder:scaffold:crdkustomizewebhookpatch # [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix. @@ -41,6 +43,7 @@ patches: #- path: patches/cainjection_in_apiserveroverloads.yaml #- path: patches/cainjection_in_eventsentropies.yaml #- path: patches/cainjection_in_consumenamespaceresources.yaml +#- path: patches/cainjection_in_cordonnodes.yaml #+kubebuilder:scaffold:crdkustomizecainjectionpatch # [WEBHOOK] To enable webhook, uncomment the following section diff --git a/config/rbac/cordonnode_editor_role.yaml b/config/rbac/cordonnode_editor_role.yaml new file mode 100644 index 0000000..f119987 --- /dev/null +++ b/config/rbac/cordonnode_editor_role.yaml @@ -0,0 +1,31 @@ +# permissions for end users to edit cordonnodes. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: cordonnode-editor-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: khaos + app.kubernetes.io/part-of: khaos + app.kubernetes.io/managed-by: kustomize + name: cordonnode-editor-role +rules: +- apiGroups: + - khaos.stackzoo.io + resources: + - cordonnodes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - khaos.stackzoo.io + resources: + - cordonnodes/status + verbs: + - get diff --git a/config/rbac/cordonnode_viewer_role.yaml b/config/rbac/cordonnode_viewer_role.yaml new file mode 100644 index 0000000..38934b2 --- /dev/null +++ b/config/rbac/cordonnode_viewer_role.yaml @@ -0,0 +1,27 @@ +# permissions for end users to view cordonnodes. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: cordonnode-viewer-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: khaos + app.kubernetes.io/part-of: khaos + app.kubernetes.io/managed-by: kustomize + name: cordonnode-viewer-role +rules: +- apiGroups: + - khaos.stackzoo.io + resources: + - cordonnodes + verbs: + - get + - list + - watch +- apiGroups: + - khaos.stackzoo.io + resources: + - cordonnodes/status + verbs: + - get diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index ce2bc19..05e4d96 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -75,25 +75,21 @@ rules: - apiGroups: - khaos.stackzoo.io resources: - - podlabelchaos + - apiserveroverloads verbs: - - create - - delete - get - list - - patch - - update - watch - apiGroups: - khaos.stackzoo.io resources: - - podlabelchaos/finalizers + - apiserveroverloads/finalizers verbs: - update - apiGroups: - khaos.stackzoo.io resources: - - podlabelchaos/status + - apiserveroverloads/status verbs: - get - patch @@ -101,21 +97,25 @@ rules: - apiGroups: - khaos.stackzoo.io resources: - - apiserveroverloads + - commandinjections verbs: + - create + - delete - get - list + - patch + - update - watch - apiGroups: - khaos.stackzoo.io resources: - - apiserveroverloads/finalizers + - commandinjections/finalizers verbs: - update - apiGroups: - khaos.stackzoo.io resources: - - apiserveroverloads/status + - commandinjections/status verbs: - get - patch @@ -123,7 +123,7 @@ rules: - apiGroups: - khaos.stackzoo.io resources: - - commandinjections + - configmapdestroyers verbs: - create - delete @@ -135,13 +135,13 @@ rules: - apiGroups: - khaos.stackzoo.io resources: - - commandinjections/finalizers + - configmapdestroyers/finalizers verbs: - update - apiGroups: - khaos.stackzoo.io resources: - - commandinjections/status + - configmapdestroyers/status verbs: - get - patch @@ -149,7 +149,7 @@ rules: - apiGroups: - khaos.stackzoo.io resources: - - configmapdestroyers + - containerresourcechaos verbs: - create - delete @@ -161,13 +161,13 @@ rules: - apiGroups: - khaos.stackzoo.io resources: - - configmapdestroyers/finalizers + - containerresourcechaos/finalizers verbs: - update - apiGroups: - khaos.stackzoo.io resources: - - configmapdestroyers/status + - containerresourcechaos/status verbs: - get - patch @@ -175,7 +175,7 @@ rules: - apiGroups: - khaos.stackzoo.io resources: - - containerresourcechaos + - cordonnodes verbs: - create - delete @@ -187,13 +187,7 @@ rules: - apiGroups: - khaos.stackzoo.io resources: - - containerresourcechaos/finalizers - verbs: - - update -- apiGroups: - - khaos.stackzoo.io - resources: - - containerresourcechaos/status + - cordonnodes/status verbs: - get - patch @@ -270,6 +264,32 @@ rules: - get - patch - update +- apiGroups: + - khaos.stackzoo.io + resources: + - podlabelchaos + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - khaos.stackzoo.io + resources: + - podlabelchaos/finalizers + verbs: + - update +- apiGroups: + - khaos.stackzoo.io + resources: + - podlabelchaos/status + verbs: + - get + - patch + - update - apiGroups: - khaos.stackzoo.io resources: diff --git a/config/samples/khaos_v1alpha1_cordonnode.yaml b/config/samples/khaos_v1alpha1_cordonnode.yaml new file mode 100644 index 0000000..18d2560 --- /dev/null +++ b/config/samples/khaos_v1alpha1_cordonnode.yaml @@ -0,0 +1,12 @@ +apiVersion: khaos.stackzoo.io/v1alpha1 +kind: CordonNode +metadata: + labels: + app.kubernetes.io/name: cordonnode + app.kubernetes.io/instance: cordonnode-sample + app.kubernetes.io/part-of: khaos + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: khaos + name: cordonnode-sample +spec: + # TODO(user): Add fields here diff --git a/config/samples/kustomization.yaml b/config/samples/kustomization.yaml index 22e6cd8..bdfa3b4 100644 --- a/config/samples/kustomization.yaml +++ b/config/samples/kustomization.yaml @@ -10,4 +10,5 @@ resources: - khaos_v1alpha1_apiserveroverload.yaml - khaos_v1alpha1_eventsentropy.yaml - khaos_v1alpha1_consumenamespaceresources.yaml +- khaos_v1alpha1_cordonnode.yaml #+kubebuilder:scaffold:manifestskustomizesamples diff --git a/examples/cordon-nodes.yaml b/examples/cordon-nodes.yaml new file mode 100644 index 0000000..18f8438 --- /dev/null +++ b/examples/cordon-nodes.yaml @@ -0,0 +1,9 @@ +apiVersion: khaos.stackzoo.io/v1alpha1 +kind: CordonNode +metadata: + name: example-cordon-node +spec: + nodesToCordon: + - test-operator-cluster-worker + - test-operator-cluster-worker2 + - test-operator-cluster-worker3 diff --git a/examples/test-node-cordon-pod.yaml b/examples/test-node-cordon-pod.yaml new file mode 100644 index 0000000..dbc05ab --- /dev/null +++ b/examples/test-node-cordon-pod.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Pod +metadata: + name: busybox-pod +spec: + containers: + - name: busybox-container + image: busybox + command: ["/bin/sh", "-c", "while true; do sleep 3600; done"] diff --git a/internal/controller/cordonnode_controller.go b/internal/controller/cordonnode_controller.go new file mode 100644 index 0000000..d7630b0 --- /dev/null +++ b/internal/controller/cordonnode_controller.go @@ -0,0 +1,81 @@ +package controller + +import ( + "context" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + khaosv1alpha1 "stackzoo.io/khaos/api/v1alpha1" +) + +// CordonNodeReconciler reconciles a CordonNode object +type CordonNodeReconciler struct { + client.Client + Log logr.Logger + Scheme *runtime.Scheme +} + +// +kubebuilder:rbac:groups=khaos.stackzoo.io,resources=cordonnodes,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=khaos.stackzoo.io,resources=cordonnodes/status,verbs=get;update;patch +// +kubebuilder:rbac:groups="",resources=nodes,verbs=get;list + +func (r *CordonNodeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + _ = r.Log.WithValues("cordonnode", req.NamespacedName) + + // Fetch the CordonNode resource + cordonNode := &khaosv1alpha1.CordonNode{} + if err := r.Get(ctx, req.NamespacedName, cordonNode); err != nil { + return reconcile.Result{}, client.IgnoreNotFound(err) + } + + // Process the CordonNode and cordon nodes + nodesCordoned, err := r.cordonNodes(cordonNode) + if err != nil { + return reconcile.Result{}, err + } + + // Update the status + cordonNode.Status.NodesCordoned = nodesCordoned + if err := r.Status().Update(ctx, cordonNode); err != nil { + return reconcile.Result{}, err + } + + return reconcile.Result{}, nil +} + +func (r *CordonNodeReconciler) cordonNodes(cordonNode *khaosv1alpha1.CordonNode) (int, error) { + nodesCordoned := 0 + + for _, nodeName := range cordonNode.Spec.NodesToCordon { + // Fetch the node + node := &corev1.Node{} + if err := r.Get(context.Background(), client.ObjectKey{Name: nodeName}, node); err != nil { + r.Log.Error(err, "Failed to fetch node", "nodeName", nodeName) + continue + } + + // Cordon the node + node.Spec.Unschedulable = true + if err := r.Update(context.Background(), node); err != nil { + r.Log.Error(err, "Failed to cordon node", "nodeName", nodeName) + continue + } + + nodesCordoned++ + } + + return nodesCordoned, nil +} + +func (r *CordonNodeReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&khaosv1alpha1.CordonNode{}). + Complete(r) +}