Added node report and associated links (#170)

* Added node report and associated links

* Fix issue displaying cluster name correctly

kube_resource_report/query.py

@@ -193,6 +193,8 @@ def map_node(_node: Node):
     node["allocatable"] = {}
     node["requests"] = new_resources()
     node["usage"] = new_resources()
+    node["pods"] = {}
+    node["slack_cost"] = 0
 
     status = _node.obj["status"]
     for k, v in status.get("capacity", {}).items():
@@ -330,6 +332,7 @@ def query_cluster(
                 user_requests[k] += v
         node_name = pod.obj["spec"].get("nodeName")
         if node_name and node_name in nodes:
+            pod_["node"] = node_name
             for k in ("cpu", "memory"):
                 nodes[node_name]["requests"][k] += pod_["requests"].get(k, 0)
         found_vpa = False
@@ -410,6 +413,11 @@ def query_cluster(
             pod["recommendation"]["memory"] * cost_per_memory,
         )
         pod["slack_cost"] = max(min(pod["cost"] - usage_cost, pod["cost"]), 0)
+        if "node" in pod.keys():
+            node_name = pod["node"]
+            if node_name and node_name in nodes:
+                node = nodes[node_name]
+                node["slack_cost"] += pod["slack_cost"]
         cluster_slack_cost += pod["slack_cost"]
 
     cluster_summary["slack_cost"] = min(cluster_cost, cluster_slack_cost)

kube_resource_report/report.py

@@ -299,6 +299,7 @@ def generate_report(
 
     applications: Dict[str, dict] = {}
     namespace_usage: Dict[tuple, dict] = {}
+    nodes: Dict[str, dict] = {}
 
     for cluster_id, summary in sorted(cluster_summaries.items()):
         for _k, pod in summary["pods"].items():
@@ -382,6 +383,12 @@ def generate_report(
             namespace["cluster"] = summary["cluster"]
             namespace_usage[(ns_pod[0], cluster_id)] = namespace
 
+        for node_name, node in summary["nodes"].items():
+            node["node_name"] = node_name
+            node["cluster"] = cluster_id
+            node["cluster_name"] = cluster_summaries[cluster_id]["cluster"].name
+            nodes[f"{cluster_id}.{node_name}"] = node
+
     if application_registry:
         resolve_application_ids(applications, application_registry)
 
@@ -429,6 +436,7 @@ def cluster_name(cluster_id):
         start,
         notifications,
         cluster_summaries,
+        nodes,
         namespace_usage,
         applications,
         teams,
@@ -456,6 +464,7 @@ def write_loading_page(out):
 def write_tsv_files(
     out: OutputManager,
     cluster_summaries,
+    nodes,
     namespace_usage,
     applications,
     teams,
@@ -503,6 +512,43 @@ def write_tsv_files(
             fields += [round(summary["slack_cost"], 2)]
             writer.writerow(fields)
 
+    with out.open("nodes.tsv") as csvfile:
+        writer = csv.writer(csvfile, delimiter="\t")
+        headers = [
+            "Cluster ID",
+            "Node",
+            "Role",
+            "Instance Type",
+            "Spot Instance",
+            "Kubelet Version",
+        ]
+        for x in resource_categories:
+            headers.extend([f"CPU {x.capitalize()}", f"Memory {x.capitalize()} [MiB]"])
+        headers.append("Cost [USD]")
+        writer.writerow(headers)
+        for _, node in sorted(nodes.items()):
+            instance_type = set()
+            kubelet_version = set()
+            if node["role"] in node_labels:
+                instance_type.add(node["instance_type"])
+            kubelet_version.add(node["kubelet_version"])
+
+            fields = [
+                node["cluster"],
+                node["node_name"],
+                node["role"],
+                node["instance_type"],
+                "Yes" if node["spot"] else "No",
+                node["kubelet_version"],
+            ]
+            for x in resource_categories:
+                fields += [
+                    round(node[x]["cpu"], 2),
+                    int(node[x]["memory"] / ONE_MEBI),
+                ]
+            fields += [round(node["cost"], 2)]
+            writer.writerow(fields)
+
     with out.open("ingresses.tsv") as csvfile:
         writer = csv.writer(csvfile, delimiter="\t")
         writer.writerow(
@@ -833,6 +879,8 @@ def write_html_files(
     context,
     alpha_ema,
     cluster_summaries,
+    nodes,
+    pods_by_node,
     teams,
     applications,
     ingresses_by_application,
@@ -846,6 +894,7 @@ def write_html_files(
     for page in [
         "index",
         "clusters",
+        "nodes",
         "ingresses",
         "routegroups",
         "teams",
@@ -868,6 +917,15 @@ def write_html_files(
         context["summary"] = summary
         out.render_template("cluster.html", context, file_name)
 
+    for node_id, node in nodes.items():
+        page = "nodes"
+        file_name = f"node-{node_id}.html"
+        context["page"] = page
+        context["cluster_id"] = cluster_id
+        context["node"] = node
+        context["pods"] = pods_by_node[node_id]
+        out.render_template("node.html", context, file_name)
+
     for team_id, team in teams.items():
         page = "teams"
         file_name = f"team-{team_id}.html"
@@ -893,6 +951,7 @@ def write_report(
     start,
     notifications,
     cluster_summaries,
+    nodes,
     namespace_usage,
     applications,
     teams,
@@ -902,7 +961,7 @@ def write_report(
     enable_routegroups: bool,
 ):
     write_tsv_files(
-        out, cluster_summaries, namespace_usage, applications, teams, node_labels
+        out, cluster_summaries, nodes, namespace_usage, applications, teams, node_labels
     )
 
     total_allocatable: dict = collections.defaultdict(int)
@@ -959,13 +1018,32 @@ def write_report(
                 }
             )
 
+    pods_by_node: Dict[str, list] = collections.defaultdict(list)
+    for cluster_id, summary in cluster_summaries.items():
+        for namespace_name, pod in summary["pods"].items():
+            namespace, name = namespace_name
+            if "node" not in pod.keys():
+                continue
+            node_name = pod["node"]
+            node_id = f"{cluster_id}.{node_name}"
+            pods_by_node[node_id].append(
+                {
+                    "cluster_id": cluster_id,
+                    "node": nodes[node_id],
+                    "namespace": namespace,
+                    "name": name,
+                    "pod": pod,
+                }
+            )
+
     total_cost = sum([s["cost"] for s in cluster_summaries.values()])
     total_hourly_cost = total_cost / HOURS_PER_MONTH
     now = datetime.datetime.utcnow()
     context = {
         "links": links,
         "notifications": notifications,
         "cluster_summaries": cluster_summaries,
+        "nodes": nodes,
         "teams": teams,
         "applications": applications,
         "namespace_usage": namespace_usage,
@@ -1012,6 +1090,8 @@ def write_report(
         context,
         alpha_ema,
         cluster_summaries,
+        nodes,
+        pods_by_node,
         teams,
         applications,
         ingresses_by_application,

kube_resource_report/templates/cluster.html

@@ -83,7 +83,7 @@ <h2 class="title is-5">Nodes</h2>
         <tbody>
             {% for node_name, node in summary.nodes.items()|sort: %}
             <tr>
-                <td>{{ node_name }}</td>
+                <td><a href="./node-{{ summary.cluster.id }}.{{ node_name }}.html">{{ node_name }}</a></td>
                 <td>{{ node.role }}</td>
                 <td>{{ node.instance_type }}</td>
                 <td>{% if node.spot: %}

kube_resource_report/templates/node.html

@@ -0,0 +1,108 @@
+{% extends "base.html" %}
+{% block title %}Node {{ node.node_name }}{% endblock %}
+{% block content %}
+<h1 class="title">Node {{ node.node_name }}
+    <span class="links">
+        {% for link in links['node']: %}
+        <a href="{{ link.href.format(cluster=summary.cluster.name, name=node_name) }}"
+           title="{{ link.title.format(cluster=summary.cluster.name, name=node_name) }}"
+           class="button {{ link.class or 'is-light' }}">
+            <span class="icon"><i class="fas fa-{{ link.icon }}"></i></span>
+        </a>
+        {% endfor %}
+    </span>
+</h1>
+<h2 class="subtitle"><a href="./cluster-{{ summary.cluster.id }}.html">{{ summary.cluster.name }}</a> / {{ node.instance_type }}</h2>
+
+<nav class="level">
+    <div class="level-item has-text-centered">
+        <div>
+            <p class="heading">Pods</p>
+            <p class="title">{{ pods|count }}</p>
+        </div>
+    </div>
+    <div class="level-item has-text-centered">
+        <div>
+            <p class="heading">CPU Requests / Allocatable</p>
+            <p class="title">{{ node.requests.cpu|cpu }} / {{ node.allocatable.cpu|cpu }}</p>
+        </div>
+    </div>
+    <div class="level-item has-text-centered">
+        <div>
+            <p class="heading">Memory Requests / Allocatable</p>
+            <p class="title">{{ node.requests.memory|filesizeformat(True) }} / {{ node.allocatable.memory|filesizeformat(True) }}</p>
+        </div>
+    </div>
+    <div class="level-item has-text-centered">
+        <div>
+            <p class="heading">Monthly Cost</p>
+            <p class="title">{{ node.cost|money }} USD</p>
+        </div>
+    </div>
+</nav>
+<div class="notification is-warning">
+    You can potentially save <strong>{{ node.slack_cost|money }} USD</strong> every month by optimizing resource requests and reducing slack.
+</div>
+
+<div class="section collapsible" data-name="pods">
+    <h2 class="title is-5">Pods</h2>
+    <table class="table is-striped is-hoverable is-fullwidth" data-sortable>
+        <thead>
+            <tr>
+                <th>Cluster</th>
+                <th>Namespace</th>
+                <th>Component</th>
+                <th>Name</th>
+                <th><abbr title="Container Images">Cont.</abbr></th>
+                <th><abbr title="CPU Requests">CR</abbr></th>
+                <th><abbr title="Memory Requests">MR</abbr></th>
+                <th>CPU</th>
+                <th>Memory (MiB)</th>
+                <th class="has-text-right">Cost</th>
+                <th class="has-text-right">Slack Cost</th>
+                {% if links['pod']: %}
+                <th></th>
+                {% endif %}
+            </tr>
+        </thead>
+        <tbody>
+            {% for row in pods: %}
+            <tr>
+                <td><a href="./cluster-{{ row.cluster_id }}.html">{{ summary.cluster.name }}</a></td>
+                <td>{{ row.namespace }}</td>
+                <td>{{ row.pod.component }}</td>
+                <td>{{ row.name }}</td>
+                <td title="{{ row.pod.container_images|join(', ') }}">{{ row.pod.container_images|count }}</td>
+                <td>{{ row.pod.requests.cpu|round(3) }}</td>
+                <td data-value="{{ row.pod.requests.memory }}">{{ row.pod.requests.memory|filesizeformat(True) }}</td>
+
+                <td style="font-size: 0.75rem" data-value="{{ row.pod.usage.cpu }}">
+                    {{ elements.resource_bar_cpu(row.pod) }}
+                </td>
+                <td style="font-size: 0.75rem" data-value="{{ row.pod.usage.memory }}">
+                    {{ elements.resource_bar_memory(row.pod) }}
+                </td>
+
+                <td class="has-text-right">{{ row.pod.cost|money }}</td>
+                <td class="has-text-right">{{ row.pod.slack_cost|money }}</td>
+
+                {% if links['pod']: %}
+                <td class="links">
+                    <div class="buttons has-addons">
+                        {% for link in links['pod']: %}
+                        <a href="{{ link.href.format(cluster=summary.cluster.name, namespace=row.namespace, name=row.name) }}"
+                           title="{{ link.title.format(cluster=summary.cluster.name, namespace=row.namespace, name=row.name) }}"
+                           class="button is-small">
+                            <span class="icon"><i class="fas fa-{{ link.icon }}"></i></span>
+                        </a>
+                        {% endfor %}
+                    </div>
+                </td>
+                {% endif %}
+            </tr>
+            {%endfor %}
+        </tbody>
+
+    </table>
+</div>
+{% endblock %}