From a68c7d4871db91edf7f7dbd69ee273cbbc8181af Mon Sep 17 00:00:00 2001 From: Kevin Barry Date: Fri, 25 Aug 2023 15:29:50 +0100 Subject: [PATCH] Add a flag to exclude tainted nodes On a cluster that has many tainted nodes it's not straightforward to understand whether there is sufficient capacity to drain one: nodes with taints can make it seem like there is more capacity than there really would be. This patch solves for the simplest use case by allowing all tainted nodes to be excluded with a flag. --- pkg/capacity/capacity.go | 15 ++++++++++--- pkg/capacity/capacity_test.go | 41 ++++++++++++++++++++++++++--------- pkg/capacity/resources.go | 3 +++ pkg/cmd/root.go | 7 ++++-- 4 files changed, 51 insertions(+), 15 deletions(-) diff --git a/pkg/capacity/capacity.go b/pkg/capacity/capacity.go index 37aefe98..579ab43f 100644 --- a/pkg/capacity/capacity.go +++ b/pkg/capacity/capacity.go @@ -29,14 +29,14 @@ import ( ) // FetchAndPrint gathers cluster resource data and outputs it -func FetchAndPrint(showContainers, showPods, showUtil, showPodCount, availableFormat bool, podLabels, nodeLabels, namespaceLabels, namespace, kubeContext, kubeConfig, output, sortBy string) { +func FetchAndPrint(showContainers, showPods, showUtil, showPodCount, excludeTainted, availableFormat bool, podLabels, nodeLabels, namespaceLabels, namespace, kubeContext, kubeConfig, output, sortBy string) { clientset, err := kube.NewClientSet(kubeContext, kubeConfig) if err != nil { fmt.Printf("Error connecting to Kubernetes: %v\n", err) os.Exit(1) } - podList, nodeList := getPodsAndNodes(clientset, podLabels, nodeLabels, namespaceLabels, namespace) + podList, nodeList := getPodsAndNodes(clientset, excludeTainted, podLabels, nodeLabels, namespaceLabels, namespace) var pmList *v1beta1.PodMetricsList var nmList *v1beta1.NodeMetricsList @@ -59,7 +59,7 @@ func FetchAndPrint(showContainers, showPods, showUtil, showPodCount, availableFo printList(&cm, showContainers, showPods, showUtil, showPodCount, showNamespace, output, sortBy, availableFormat) } -func getPodsAndNodes(clientset kubernetes.Interface, podLabels, nodeLabels, namespaceLabels, namespace string) (*corev1.PodList, *corev1.NodeList) { +func getPodsAndNodes(clientset kubernetes.Interface, excludeTainted bool, podLabels, nodeLabels, namespaceLabels, namespace string) (*corev1.PodList, *corev1.NodeList) { nodeList, err := clientset.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{ LabelSelector: nodeLabels, }) @@ -67,6 +67,15 @@ func getPodsAndNodes(clientset kubernetes.Interface, podLabels, nodeLabels, name fmt.Printf("Error listing Nodes: %v\n", err) os.Exit(2) } + if excludeTainted { + filteredNodeList := []corev1.Node{} + for _, node := range nodeList.Items { + if len(node.Spec.Taints) == 0 { + filteredNodeList = append(filteredNodeList, node) + } + } + nodeList.Items = filteredNodeList + } podList, err := clientset.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{ LabelSelector: podLabels, diff --git a/pkg/capacity/capacity_test.go b/pkg/capacity/capacity_test.go index a78ba3c3..5a92025e 100644 --- a/pkg/capacity/capacity_test.go +++ b/pkg/capacity/capacity_test.go @@ -25,8 +25,8 @@ import ( func TestGetPodsAndNodes(t *testing.T) { clientset := fake.NewSimpleClientset( - node("mynode", map[string]string{"hello": "world"}), - node("mynode2", map[string]string{"hello": "world", "moon": "lol"}), + node("mynode", map[string]string{"hello": "world"}, false), + node("mynode2", map[string]string{"hello": "world", "moon": "lol"}, true), namespace("default", map[string]string{"app": "true"}), namespace("kube-system", map[string]string{"system": "true"}), namespace("other", map[string]string{"app": "true", "system": "true"}), @@ -40,7 +40,7 @@ func TestGetPodsAndNodes(t *testing.T) { pod("mynode", "default", "mypod6", map[string]string{"g": "test"}), ) - podList, nodeList := getPodsAndNodes(clientset, "", "", "", "") + podList, nodeList := getPodsAndNodes(clientset, false, "", "", "", "") assert.Equal(t, []string{"mynode", "mynode2"}, listNodes(nodeList)) assert.Equal(t, []string{ "another/mypod5", @@ -52,7 +52,16 @@ func TestGetPodsAndNodes(t *testing.T) { "other/mypod3", }, listPods(podList)) - podList, nodeList = getPodsAndNodes(clientset, "", "hello=world", "", "") + podList, nodeList = getPodsAndNodes(clientset, true, "", "hello=world", "", "") + assert.Equal(t, []string{"mynode"}, listNodes(nodeList)) + assert.Equal(t, []string{ + "another/mypod5", + "default/mypod", + "default/mypod6", + "other/mypod2", + }, listPods(podList)) + + podList, nodeList = getPodsAndNodes(clientset, false, "", "hello=world", "", "") assert.Equal(t, []string{"mynode", "mynode2"}, listNodes(nodeList)) assert.Equal(t, []string{ "another/mypod5", @@ -64,7 +73,7 @@ func TestGetPodsAndNodes(t *testing.T) { "other/mypod3", }, listPods(podList)) - podList, nodeList = getPodsAndNodes(clientset, "", "moon=lol", "", "") + podList, nodeList = getPodsAndNodes(clientset, false, "", "moon=lol", "", "") assert.Equal(t, []string{"mynode2"}, listNodes(nodeList)) assert.Equal(t, []string{ "default/mypod4", @@ -72,27 +81,27 @@ func TestGetPodsAndNodes(t *testing.T) { "other/mypod3", }, listPods(podList)) - podList, nodeList = getPodsAndNodes(clientset, "a=test", "", "", "") + podList, nodeList = getPodsAndNodes(clientset, false, "a=test", "", "", "") assert.Equal(t, []string{"mynode", "mynode2"}, listNodes(nodeList)) assert.Equal(t, []string{ "default/mypod", }, listPods(podList)) - podList, nodeList = getPodsAndNodes(clientset, "a=test,b!=test", "", "app=true", "") + podList, nodeList = getPodsAndNodes(clientset, false, "a=test,b!=test", "", "app=true", "") assert.Equal(t, []string{"mynode", "mynode2"}, listNodes(nodeList)) assert.Equal(t, []string{ "default/mypod", }, listPods(podList)) - podList, nodeList = getPodsAndNodes(clientset, "a=test,b!=test", "", "", "default") + podList, nodeList = getPodsAndNodes(clientset, false, "a=test,b!=test", "", "", "default") assert.Equal(t, []string{"mynode", "mynode2"}, listNodes(nodeList)) assert.Equal(t, []string{ "default/mypod", }, listPods(podList)) } -func node(name string, labels map[string]string) *corev1.Node { - return &corev1.Node{ +func node(name string, labels map[string]string, tainted bool) *corev1.Node { + n := &corev1.Node{ TypeMeta: metav1.TypeMeta{ Kind: "Node", APIVersion: "v1", @@ -102,6 +111,18 @@ func node(name string, labels map[string]string) *corev1.Node { Labels: labels, }, } + if tainted { + n.Spec = corev1.NodeSpec{ + Taints: []corev1.Taint{ + { + Key: "taint", + Value: "true", + Effect: corev1.TaintEffectNoSchedule, + }, + }, + } + } + return n } func namespace(name string, labels map[string]string) *corev1.Namespace { diff --git a/pkg/capacity/resources.go b/pkg/capacity/resources.go index 74b5e74f..90946066 100644 --- a/pkg/capacity/resources.go +++ b/pkg/capacity/resources.go @@ -129,6 +129,9 @@ func buildClusterMetric(podList *corev1.PodList, pmList *v1beta1.PodMetricsList, if nmList != nil { for _, nm := range nmList.Items { + if cm.nodeMetrics[nm.Name] == nil { + continue + } cm.nodeMetrics[nm.Name].cpu.utilization = nm.Usage["cpu"] cm.nodeMetrics[nm.Name].memory.utilization = nm.Usage["memory"] } diff --git a/pkg/cmd/root.go b/pkg/cmd/root.go index 3d39d050..0c23c5ac 100644 --- a/pkg/cmd/root.go +++ b/pkg/cmd/root.go @@ -28,6 +28,7 @@ var showUtil bool var showPodCount bool var podLabels string var nodeLabels string +var excludeTainted bool var namespaceLabels string var namespace string var kubeContext string @@ -50,8 +51,8 @@ var rootCmd = &cobra.Command{ os.Exit(1) } - capacity.FetchAndPrint(showContainers, showPods, showUtil, showPodCount, availableFormat, podLabels, nodeLabels, - namespaceLabels, namespace, kubeContext, kubeConfig, outputFormat, sortBy) + capacity.FetchAndPrint(showContainers, showPods, showUtil, showPodCount, excludeTainted, availableFormat, podLabels, + nodeLabels, namespaceLabels, namespace, kubeContext, kubeConfig, outputFormat, sortBy) }, } @@ -70,6 +71,8 @@ func init() { "pod-labels", "l", "", "labels to filter pods with") rootCmd.PersistentFlags().StringVarP(&nodeLabels, "node-labels", "", "", "labels to filter nodes with") + rootCmd.PersistentFlags().BoolVarP(&excludeTainted, + "no-taint", "", false, "exclude nodes with taints") rootCmd.PersistentFlags().StringVarP(&namespaceLabels, "namespace-labels", "", "", "labels to filter namespaces with") rootCmd.PersistentFlags().StringVarP(&namespace,