From 02e29cf7f3e3cb25bb126f9493d5bf8330a4c2e6 Mon Sep 17 00:00:00 2001 From: Gavin Bunney <409207+gavinbunney@users.noreply.github.com> Date: Wed, 1 Jul 2020 12:59:39 -0700 Subject: [PATCH] Added node report and associated links (#170) * Added node report and associated links * Fix issue displaying cluster name correctly --- kube_resource_report/query.py | 8 ++ kube_resource_report/report.py | 82 ++++++++++++- kube_resource_report/templates/cluster.html | 2 +- kube_resource_report/templates/node.html | 108 ++++++++++++++++++ kube_resource_report/templates/nodes.html | 91 +++++++++++++++ .../templates/partials/navbar.html | 1 + tests/test_query.py | 4 + 7 files changed, 294 insertions(+), 2 deletions(-) create mode 100644 kube_resource_report/templates/node.html create mode 100644 kube_resource_report/templates/nodes.html diff --git a/kube_resource_report/query.py b/kube_resource_report/query.py index eb342e9..429bfc5 100644 --- a/kube_resource_report/query.py +++ b/kube_resource_report/query.py @@ -193,6 +193,8 @@ def map_node(_node: Node): node["allocatable"] = {} node["requests"] = new_resources() node["usage"] = new_resources() + node["pods"] = {} + node["slack_cost"] = 0 status = _node.obj["status"] for k, v in status.get("capacity", {}).items(): @@ -330,6 +332,7 @@ def query_cluster( user_requests[k] += v node_name = pod.obj["spec"].get("nodeName") if node_name and node_name in nodes: + pod_["node"] = node_name for k in ("cpu", "memory"): nodes[node_name]["requests"][k] += pod_["requests"].get(k, 0) found_vpa = False @@ -410,6 +413,11 @@ def query_cluster( pod["recommendation"]["memory"] * cost_per_memory, ) pod["slack_cost"] = max(min(pod["cost"] - usage_cost, pod["cost"]), 0) + if "node" in pod.keys(): + node_name = pod["node"] + if node_name and node_name in nodes: + node = nodes[node_name] + node["slack_cost"] += pod["slack_cost"] cluster_slack_cost += pod["slack_cost"] cluster_summary["slack_cost"] = min(cluster_cost, cluster_slack_cost) diff --git a/kube_resource_report/report.py b/kube_resource_report/report.py index 998b6d9..e68cda7 100755 --- a/kube_resource_report/report.py +++ b/kube_resource_report/report.py @@ -299,6 +299,7 @@ def generate_report( applications: Dict[str, dict] = {} namespace_usage: Dict[tuple, dict] = {} + nodes: Dict[str, dict] = {} for cluster_id, summary in sorted(cluster_summaries.items()): for _k, pod in summary["pods"].items(): @@ -382,6 +383,12 @@ def generate_report( namespace["cluster"] = summary["cluster"] namespace_usage[(ns_pod[0], cluster_id)] = namespace + for node_name, node in summary["nodes"].items(): + node["node_name"] = node_name + node["cluster"] = cluster_id + node["cluster_name"] = cluster_summaries[cluster_id]["cluster"].name + nodes[f"{cluster_id}.{node_name}"] = node + if application_registry: resolve_application_ids(applications, application_registry) @@ -429,6 +436,7 @@ def cluster_name(cluster_id): start, notifications, cluster_summaries, + nodes, namespace_usage, applications, teams, @@ -456,6 +464,7 @@ def write_loading_page(out): def write_tsv_files( out: OutputManager, cluster_summaries, + nodes, namespace_usage, applications, teams, @@ -503,6 +512,43 @@ def write_tsv_files( fields += [round(summary["slack_cost"], 2)] writer.writerow(fields) + with out.open("nodes.tsv") as csvfile: + writer = csv.writer(csvfile, delimiter="\t") + headers = [ + "Cluster ID", + "Node", + "Role", + "Instance Type", + "Spot Instance", + "Kubelet Version", + ] + for x in resource_categories: + headers.extend([f"CPU {x.capitalize()}", f"Memory {x.capitalize()} [MiB]"]) + headers.append("Cost [USD]") + writer.writerow(headers) + for _, node in sorted(nodes.items()): + instance_type = set() + kubelet_version = set() + if node["role"] in node_labels: + instance_type.add(node["instance_type"]) + kubelet_version.add(node["kubelet_version"]) + + fields = [ + node["cluster"], + node["node_name"], + node["role"], + node["instance_type"], + "Yes" if node["spot"] else "No", + node["kubelet_version"], + ] + for x in resource_categories: + fields += [ + round(node[x]["cpu"], 2), + int(node[x]["memory"] / ONE_MEBI), + ] + fields += [round(node["cost"], 2)] + writer.writerow(fields) + with out.open("ingresses.tsv") as csvfile: writer = csv.writer(csvfile, delimiter="\t") writer.writerow( @@ -833,6 +879,8 @@ def write_html_files( context, alpha_ema, cluster_summaries, + nodes, + pods_by_node, teams, applications, ingresses_by_application, @@ -846,6 +894,7 @@ def write_html_files( for page in [ "index", "clusters", + "nodes", "ingresses", "routegroups", "teams", @@ -868,6 +917,15 @@ def write_html_files( context["summary"] = summary out.render_template("cluster.html", context, file_name) + for node_id, node in nodes.items(): + page = "nodes" + file_name = f"node-{node_id}.html" + context["page"] = page + context["cluster_id"] = cluster_id + context["node"] = node + context["pods"] = pods_by_node[node_id] + out.render_template("node.html", context, file_name) + for team_id, team in teams.items(): page = "teams" file_name = f"team-{team_id}.html" @@ -893,6 +951,7 @@ def write_report( start, notifications, cluster_summaries, + nodes, namespace_usage, applications, teams, @@ -902,7 +961,7 @@ def write_report( enable_routegroups: bool, ): write_tsv_files( - out, cluster_summaries, namespace_usage, applications, teams, node_labels + out, cluster_summaries, nodes, namespace_usage, applications, teams, node_labels ) total_allocatable: dict = collections.defaultdict(int) @@ -959,6 +1018,24 @@ def write_report( } ) + pods_by_node: Dict[str, list] = collections.defaultdict(list) + for cluster_id, summary in cluster_summaries.items(): + for namespace_name, pod in summary["pods"].items(): + namespace, name = namespace_name + if "node" not in pod.keys(): + continue + node_name = pod["node"] + node_id = f"{cluster_id}.{node_name}" + pods_by_node[node_id].append( + { + "cluster_id": cluster_id, + "node": nodes[node_id], + "namespace": namespace, + "name": name, + "pod": pod, + } + ) + total_cost = sum([s["cost"] for s in cluster_summaries.values()]) total_hourly_cost = total_cost / HOURS_PER_MONTH now = datetime.datetime.utcnow() @@ -966,6 +1043,7 @@ def write_report( "links": links, "notifications": notifications, "cluster_summaries": cluster_summaries, + "nodes": nodes, "teams": teams, "applications": applications, "namespace_usage": namespace_usage, @@ -1012,6 +1090,8 @@ def write_report( context, alpha_ema, cluster_summaries, + nodes, + pods_by_node, teams, applications, ingresses_by_application, diff --git a/kube_resource_report/templates/cluster.html b/kube_resource_report/templates/cluster.html index c695fdd..1f52f88 100644 --- a/kube_resource_report/templates/cluster.html +++ b/kube_resource_report/templates/cluster.html @@ -83,7 +83,7 @@

Nodes

{% for node_name, node in summary.nodes.items()|sort: %} - {{ node_name }} + {{ node_name }} {{ node.role }} {{ node.instance_type }} {% if node.spot: %} diff --git a/kube_resource_report/templates/node.html b/kube_resource_report/templates/node.html new file mode 100644 index 0000000..e4e852e --- /dev/null +++ b/kube_resource_report/templates/node.html @@ -0,0 +1,108 @@ +{% extends "base.html" %} +{% block title %}Node {{ node.node_name }}{% endblock %} +{% block content %} +

Node {{ node.node_name }} + + {% for link in links['node']: %} + + + + {% endfor %} + +

+

{{ summary.cluster.name }} / {{ node.instance_type }}

+ + +
+ You can potentially save {{ node.slack_cost|money }} USD every month by optimizing resource requests and reducing slack. +
+ +
+

Pods

+ + + + + + + + + + + + + + + {% if links['pod']: %} + + {% endif %} + + + + {% for row in pods: %} + + + + + + + + + + + + + + + + {% if links['pod']: %} + + {% endif %} + + {%endfor %} + + +
ClusterNamespaceComponentNameCont.CRMRCPUMemory (MiB)CostSlack Cost
{{ summary.cluster.name }}{{ row.namespace }}{{ row.pod.component }}{{ row.name }}{{ row.pod.container_images|count }}{{ row.pod.requests.cpu|round(3) }}{{ row.pod.requests.memory|filesizeformat(True) }} + {{ elements.resource_bar_cpu(row.pod) }} + + {{ elements.resource_bar_memory(row.pod) }} + {{ row.pod.cost|money }}{{ row.pod.slack_cost|money }} +
+ {% for link in links['pod']: %} + + + + {% endfor %} +
+
+
+{% endblock %} diff --git a/kube_resource_report/templates/nodes.html b/kube_resource_report/templates/nodes.html new file mode 100644 index 0000000..648ad05 --- /dev/null +++ b/kube_resource_report/templates/nodes.html @@ -0,0 +1,91 @@ +{% extends "base.html" %} +{% block title %}Nodes{% endblock %} +{% block content %} + +

All Nodes + + + + + +

+ + + + + + + + + + + + + + + + {% if links['node']: %} + + {% endif %} + + + + {% for node_id, node in nodes.items()|sort: %} + + + + + + + + + + + + + + + {% if links['node']: %} + + {% endif %} + + {%endfor %} + + +
ClusterNameRoleInstance TypeS?VersionCCMCCPUMemory (GiB)Cost
{{ node.cluster_name }}{{ node.node_name }}{{ node.role }}{{ node.instance_type }}{% if node.spot: %} + + {% endif %} + {{ node.kubelet_version }}{{ node.capacity.cpu|round(1) }}{{ node.capacity.memory|filesizeformat(True) }} +
+ {{ node.usage.cpu|round(1) }} + {{ node.requests.cpu|round(1) }} + {{ node.allocatable.cpu|round(1) }} +
+
+ + +
+ 		+
+ {{ node.usage.memory|memory('GiB') }} + {{ node.requests.memory|memory('GiB') }} + {{ node.allocatable.memory|memory('GiB') }} +
+
+ + +
+ 		{{ node.cost|money }} +
+ {% for link in links['node']: %} + + + + {% endfor %} +
+
+{% endblock %} diff --git a/kube_resource_report/templates/partials/navbar.html b/kube_resource_report/templates/partials/navbar.html index 056e78e..cb0fc90 100644 --- a/kube_resource_report/templates/partials/navbar.html +++ b/kube_resource_report/templates/partials/navbar.html @@ -10,6 +10,7 @@
Overview Clusters + Nodes Ingresses {% if enable_routegroups: %} RouteGroups diff --git a/tests/test_query.py b/tests/test_query.py index e37c2ba..80ee25f 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -13,10 +13,12 @@ def test_map_empty_node(): "usage": {"cpu": 0, "memory": 0}, "requests": {"cpu": 0, "memory": 0}, "cost": 0, + "slack_cost": 0, "instance_type": "unknown", "kubelet_version": "", "role": "worker", "spot": False, + "pods": {}, } @@ -39,10 +41,12 @@ def test_map_gke_preemptible_node(): "usage": {"cpu": 0, "memory": 0}, "requests": {"cpu": 0, "memory": 0}, "cost": 0, + "slack_cost": 0, "instance_type": "1-standard-2-preemptible", "kubelet_version": "", "role": "worker", "spot": True, + "pods": {}, }