From 89d1bdbce4ae80bf45e77a7ababf71814617996d Mon Sep 17 00:00:00 2001 From: John McGrath <8764013+jmcgrath207@users.noreply.github.com> Date: Wed, 31 Jan 2024 02:08:06 -0600 Subject: [PATCH] bug: fixed metric eviction for all metric types (#61) --- Header.md | 10 +- Makefile | 4 +- README.md | 15 ++- chart/Chart.yaml | 4 +- chart/README.md | 5 +- chart/index.yaml | 24 +++- chart/values.yaml | 2 +- main.go | 286 ++++++++++++++++++---------------------------- 8 files changed, 158 insertions(+), 192 deletions(-) diff --git a/Header.md b/Header.md index 3d3def1..b90f4e0 100644 --- a/Header.md +++ b/Header.md @@ -1,14 +1,16 @@ -# K8s Ephemeral Storage Metrics. +# K8s Ephemeral Storage Metrics [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) [![Actions Status](https://github.com/jmcgrath207/k8s-ephemeral-storage-metrics/workflows/ci/badge.svg)](https://github.com/jmcgrath207/k8s-ephemeral-storage-metrics/actions) [![Artifact Hub](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/k8s-ephemeral-storage-metrics)](https://artifacthub.io/packages/helm/k8s-ephemeral-storage-metrics/k8s-ephemeral-storage-metrics) -The goal of this project is to export ephemeral storage metric usage per pod to Prometheus that is address in this -issue [Here](https://github.com/kubernetes/kubernetes/issues/69507) +A prometheus ephemeral storage metric exporter for pods, containers, +nodes, and volumes. -It accomplishes this by creating a Service Monitor +This project was created +to address lack of monitoring in [Kubernetes](https://github.com/kubernetes/kubernetes/issues/69507) +This project does not monitor CSI backed ephemeral storage ex. [Generic ephemeral volumes](https://kubernetes.io/docs/concepts/storage/ephemeral-volumes/#generic-ephemeral-volumes) ![main image](img/screenshot.png) diff --git a/Makefile b/Makefile index d2e24f5..c507f0c 100644 --- a/Makefile +++ b/Makefile @@ -58,10 +58,10 @@ release-helm: cd .. release: github_login release-docker release-helm helm-docs - # ex. make VERSION=1.4.4 release + # ex. make VERSION=1.5.0 release release-github: github_login - # ex. make VERSION=1.4.4 release-github + # ex. make VERSION=1.5.0 release-github gh release create ${VERSION} --generate-notes gh release upload ${VERSION} "chart/k8s-ephemeral-storage-metrics-${VERSION}.tgz" rm chart/k8s-ephemeral-storage-metrics-*.tgz diff --git a/README.md b/README.md index a85cb3d..f54c663 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,16 @@ -# K8s Ephemeral Storage Metrics. +# K8s Ephemeral Storage Metrics [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) [![Actions Status](https://github.com/jmcgrath207/k8s-ephemeral-storage-metrics/workflows/ci/badge.svg)](https://github.com/jmcgrath207/k8s-ephemeral-storage-metrics/actions) [![Artifact Hub](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/k8s-ephemeral-storage-metrics)](https://artifacthub.io/packages/helm/k8s-ephemeral-storage-metrics/k8s-ephemeral-storage-metrics) -The goal of this project is to export ephemeral storage metric usage per pod to Prometheus that is address in this -issue [Here](https://github.com/kubernetes/kubernetes/issues/69507) +A prometheus ephemeral storage metric exporter for pods, containers, +nodes, and volumes. -It accomplishes this by creating a Service Monitor +This project was created +to address lack of monitoring in [Kubernetes](https://github.com/kubernetes/kubernetes/issues/69507) +This project does not monitor CSI backed ephemeral storage ex. [Generic ephemeral volumes](https://kubernetes.io/docs/concepts/storage/ephemeral-volumes/#generic-ephemeral-volumes) ![main image](img/screenshot.png) @@ -30,13 +32,14 @@ helm upgrade --install my-deployment k8s-ephemeral-storage-metrics/k8s-ephemeral | dev | object | `{"enabled":false,"image":{"imagePullPolicy":"IfNotPresent"}}` | For local development of kind and/or deploy grow and shrink test pods | | image.imagePullPolicy | string | `"IfNotPresent"` | | | image.repository | string | `"ghcr.io/jmcgrath207/k8s-ephemeral-storage-metrics"` | | -| image.tag | string | `"1.4.4"` | | +| image.tag | string | `"1.5.0"` | | | interval | int | `15` | Polling node rate for exporter | | log_level | string | `"info"` | | | max_node_concurrency | int | `10` | Max number of concurrent query requests to the kubernetes API. | -| metrics | object | `{"adjusted_polling_rate":false,"ephemeral_storage_container_limit_percentage":true,"ephemeral_storage_node_available":true,"ephemeral_storage_node_capacity":true,"ephemeral_storage_node_percentage":true,"ephemeral_storage_pod_usage":true}` | Set metrics you want to enable | +| metrics | object | `{"adjusted_polling_rate":false,"ephemeral_storage_container_limit_percentage":true,"ephemeral_storage_container_volume_limit_percentage":true,"ephemeral_storage_node_available":true,"ephemeral_storage_node_capacity":true,"ephemeral_storage_node_percentage":true,"ephemeral_storage_pod_usage":true}` | Set metrics you want to enable | | metrics.adjusted_polling_rate | bool | `false` | Create the ephemeral_storage_adjusted_polling_rate metrics to report Adjusted Poll Rate in milliseconds. Typically used for testing. | | metrics.ephemeral_storage_container_limit_percentage | bool | `true` | Percentage of ephemeral storage used by a container in a pod | +| metrics.ephemeral_storage_container_volume_limit_percentage | bool | `true` | Percentage of ephemeral storage used by a container's volume in a pod | | metrics.ephemeral_storage_node_available | bool | `true` | Available ephemeral storage for a node | | metrics.ephemeral_storage_node_capacity | bool | `true` | Capacity of ephemeral storage for a node | | metrics.ephemeral_storage_node_percentage | bool | `true` | Percentage of ephemeral storage used on a node | diff --git a/chart/Chart.yaml b/chart/Chart.yaml index 54c5335..c24b513 100644 --- a/chart/Chart.yaml +++ b/chart/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: k8s-ephemeral-storage-metrics -version: 1.4.4 -appVersion: 1.4.4 +version: 1.5.0 +appVersion: 1.5.0 kubeVersion: ">=1.21.0-0" description: Ephemeral storage metrics for prometheus operator. home: https://github.com/jmcgrath207/k8s-ephemeral-storage-metrics diff --git a/chart/README.md b/chart/README.md index b2ed9e6..40d243a 100644 --- a/chart/README.md +++ b/chart/README.md @@ -15,13 +15,14 @@ helm upgrade --install my-deployment k8s-ephemeral-storage-metrics/k8s-ephemeral | dev | object | `{"enabled":false,"image":{"imagePullPolicy":"IfNotPresent"}}` | For local development of kind and/or deploy grow and shrink test pods | | image.imagePullPolicy | string | `"IfNotPresent"` | | | image.repository | string | `"ghcr.io/jmcgrath207/k8s-ephemeral-storage-metrics"` | | -| image.tag | string | `"1.4.4"` | | +| image.tag | string | `"1.5.0"` | | | interval | int | `15` | Polling node rate for exporter | | log_level | string | `"info"` | | | max_node_concurrency | int | `10` | Max number of concurrent query requests to the kubernetes API. | -| metrics | object | `{"adjusted_polling_rate":false,"ephemeral_storage_container_limit_percentage":true,"ephemeral_storage_node_available":true,"ephemeral_storage_node_capacity":true,"ephemeral_storage_node_percentage":true,"ephemeral_storage_pod_usage":true}` | Set metrics you want to enable | +| metrics | object | `{"adjusted_polling_rate":false,"ephemeral_storage_container_limit_percentage":true,"ephemeral_storage_container_volume_limit_percentage":true,"ephemeral_storage_node_available":true,"ephemeral_storage_node_capacity":true,"ephemeral_storage_node_percentage":true,"ephemeral_storage_pod_usage":true}` | Set metrics you want to enable | | metrics.adjusted_polling_rate | bool | `false` | Create the ephemeral_storage_adjusted_polling_rate metrics to report Adjusted Poll Rate in milliseconds. Typically used for testing. | | metrics.ephemeral_storage_container_limit_percentage | bool | `true` | Percentage of ephemeral storage used by a container in a pod | +| metrics.ephemeral_storage_container_volume_limit_percentage | bool | `true` | Percentage of ephemeral storage used by a container's volume in a pod | | metrics.ephemeral_storage_node_available | bool | `true` | Available ephemeral storage for a node | | metrics.ephemeral_storage_node_capacity | bool | `true` | Capacity of ephemeral storage for a node | | metrics.ephemeral_storage_node_percentage | bool | `true` | Percentage of ephemeral storage used on a node | diff --git a/chart/index.yaml b/chart/index.yaml index 1d1a12a..0d4810e 100644 --- a/chart/index.yaml +++ b/chart/index.yaml @@ -1,6 +1,28 @@ apiVersion: v1 entries: k8s-ephemeral-storage-metrics: + - annotations: + artifacthub.io/license: MIT + artifacthub.io/links: | + - name: Documentation + url: https://github.com/jmcgrath207/k8s-ephemeral-storage-metrics + artifacthub.io/prerelease: "false" + apiVersion: v2 + appVersion: 1.5.0 + created: "2024-01-31T02:02:15.868143053-06:00" + description: Ephemeral storage metrics for prometheus operator. + digest: defc03c7f3fdd6614c440eecb10df350f835b61d0df78c8e9977dafcfa37d5f8 + home: https://github.com/jmcgrath207/k8s-ephemeral-storage-metrics + keywords: + - kubernetes + - metrics + kubeVersion: '>=1.21.0-0' + name: k8s-ephemeral-storage-metrics + sources: + - https://github.com/jmcgrath207/k8s-ephemeral-storage-metrics + urls: + - https://github.com/jmcgrath207/k8s-ephemeral-storage-metrics/releases/download/1.5.0/k8s-ephemeral-storage-metrics-1.5.0.tgz + version: 1.5.0 - annotations: artifacthub.io/license: MIT artifacthub.io/links: | @@ -309,4 +331,4 @@ entries: urls: - https://github.com/jmcgrath207/k8s-ephemeral-storage-metrics/releases/download/1.0.0/k8s-ephemeral-storage-metrics-1.0.0.tgz version: 1.0.0 -generated: "2024-01-23T21:24:50.71281414-06:00" +generated: "2024-01-31T02:02:15.867765844-06:00" diff --git a/chart/values.yaml b/chart/values.yaml index b9b1818..b49d7f9 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -1,6 +1,6 @@ image: repository: ghcr.io/jmcgrath207/k8s-ephemeral-storage-metrics - tag: 1.4.4 + tag: 1.5.0 imagePullPolicy: IfNotPresent # -- Set metrics you want to enable diff --git a/main.go b/main.go index 0bb2325..09848a4 100644 --- a/main.go +++ b/main.go @@ -50,7 +50,7 @@ var ( nodeCapacityGaugeVec *prometheus.GaugeVec nodePercentageGaugeVec *prometheus.GaugeVec containerPercentageLimitsVec *prometheus.GaugeVec - containerVolumePercentageLimitsVec *prometheus.GaugeVec + containerPercentageVolumeLimitsVec *prometheus.GaugeVec nodeSlice []string maxNodeConcurrency int podResourceLookup map[string]pod @@ -244,11 +244,11 @@ func podWatch() { getPodData(*p) }, DeleteFunc: func(obj interface{}) { + p := obj.(*v1.Pod) podResourceLookupMutex.Lock() - delete(podResourceLookup, obj.(*v1.Pod).Name) + delete(podResourceLookup, p.Name) podResourceLookupMutex.Unlock() - // TODO: remove pod from prom metrics. - // ex. podGaugeVec.DeletePartialMatch(prometheus.Labels{"node_name": nodeName}) + evictPodFromMetrics(*p) }, } @@ -278,6 +278,15 @@ func podWatch() { } +func evictPodFromMetrics(p v1.Pod) { + + podGaugeVec.DeletePartialMatch(prometheus.Labels{"pod_name": p.Name}) + for _, c := range p.Spec.Containers { + containerPercentageLimitsVec.DeletePartialMatch(prometheus.Labels{"container": c.Name}) + containerPercentageVolumeLimitsVec.DeletePartialMatch(prometheus.Labels{"container": c.Name}) + } +} + func getNodes() { oldNodeSet := mapset.NewSet[string]() nodeSet := mapset.NewSet[string]() @@ -308,7 +317,9 @@ func getNodes() { // Evict Metrics where the node doesn't exist anymore. for _, deadNode := range deadNodesSet.ToSlice() { - podGaugeVec.DeletePartialMatch(prometheus.Labels{"node_name": deadNode}) + nodeAvailableGaugeVec.DeletePartialMatch(prometheus.Labels{"node_name": deadNode}) + nodeCapacityGaugeVec.DeletePartialMatch(prometheus.Labels{"node_name": deadNode}) + nodePercentageGaugeVec.DeletePartialMatch(prometheus.Labels{"node_name": deadNode}) log.Info().Msgf("Node %s does not exist. Removing from monitoring", deadNode) } @@ -345,15 +356,9 @@ func queryNode(node string) ([]byte, error) { } -type CollectMetric struct { - value float64 - name string - labels prometheus.Labels -} - -func generateLabels(podName string, podNamespace string, nodeName string, usedBytes float64, availableBytes float64, capacityBytes float64, volumes []volume) []CollectMetric { +func createMetricsValues(podName string, podNamespace string, nodeName string, usedBytes float64, availableBytes float64, capacityBytes float64, volumes []volume) { - var labelsList []CollectMetric + var setValue float64 podResourceLookupMutex.RLock() podResult, okPodResult := podResourceLookup[podName] podResourceLookupMutex.RUnlock() @@ -361,7 +366,8 @@ func generateLabels(podName string, podNamespace string, nodeName string, usedBy // TODO: something seems wrong about the metrics. // the volume capacityBytes is not reflected in this query // kubectl get --raw "/api/v1/nodes/ephemeral-metrics-cluster-worker/proxy/stats/summary" - // might need to source it from the pod spec + // need to source it from the pod spec + // make issue upstream with CA advisor // TODO: need to do a grow and shrink test for this. if ephemeralStorageContainerVolumeLimitsPercentage { // TODO: what a mess...need to figure out a better way. @@ -374,11 +380,7 @@ func generateLabels(podName string, podNamespace string, nodeName string, usedBy labels := prometheus.Labels{"pod_namespace": podNamespace, "pod_name": podName, "node_name": nodeName, "container": c.name, "volume_name": v.Name, "mount_path": edv.mountPath} - labelsList = append(labelsList, CollectMetric{ - value: (float64(v.UsedBytes) / edv.sizeLimit) * 100.0, - name: "ephemeral_storage_container_volume_limit_percentage", - labels: labels, - }) + containerPercentageVolumeLimitsVec.With(labels).Set((float64(v.UsedBytes) / edv.sizeLimit) * 100.0) } } } @@ -394,67 +396,42 @@ func generateLabels(podName string, podNamespace string, nodeName string, usedBy "pod_name": podName, "node_name": nodeName, "container": c.name} if c.limit != 0 { // Use Limit from Container - labelsList = append(labelsList, CollectMetric{ - value: (usedBytes / c.limit) * 100.0, - name: "ephemeral_storage_container_limit_percentage", - labels: labels, - }) + setValue = (usedBytes / c.limit) * 100.0 } else { // Default to Node Available Ephemeral Storage - labelsList = append(labelsList, CollectMetric{ - value: (availableBytes / capacityBytes) * 100.0, - name: "ephemeral_storage_container_limit_percentage", - labels: labels, - }) + setValue = (availableBytes / capacityBytes) * 100.0 } + containerPercentageLimitsVec.With(labels).Set(setValue) } } } if ephemeralStoragePodUsage { - labelsList = append(labelsList, CollectMetric{ - value: usedBytes, - name: "ephemeral_storage_pod_usage", - labels: prometheus.Labels{"pod_namespace": podNamespace, - "pod_name": podName, "node_name": nodeName}, - }) + labels := prometheus.Labels{"pod_namespace": podNamespace, + "pod_name": podName, "node_name": nodeName} + podGaugeVec.With(labels).Set(usedBytes) log.Debug().Msg(fmt.Sprintf("pod %s/%s on %s with usedBytes: %f", podNamespace, podName, nodeName, usedBytes)) } if ephemeralStorageNodeAvailable { - labelsList = append(labelsList, CollectMetric{ - value: availableBytes, - name: "ephemeral_storage_node_available", - labels: prometheus.Labels{"node_name": nodeName}}, - ) + nodeAvailableGaugeVec.With(prometheus.Labels{"node_name": nodeName}).Set(availableBytes) log.Debug().Msg(fmt.Sprintf("Node: %s availble bytes: %f", nodeName, availableBytes)) } if ephemeralStorageNodeCapacity { - labelsList = append(labelsList, CollectMetric{ - value: capacityBytes, - name: "ephemeral_storage_node_capacity", - labels: prometheus.Labels{"node_name": nodeName}}, - ) + nodeCapacityGaugeVec.With(prometheus.Labels{"node_name": nodeName}).Set(capacityBytes) log.Debug().Msg(fmt.Sprintf("Node: %s capacity bytes: %f", nodeName, capacityBytes)) } if ephemeralStorageNodePercentage { - percentage := (availableBytes / capacityBytes) * 100.0 - labelsList = append(labelsList, CollectMetric{ - value: percentage, - name: "ephemeral_storage_node_percentage", - labels: prometheus.Labels{"node_name": nodeName}}, - ) - log.Debug().Msg(fmt.Sprintf("Node: %s percentage used: %f", nodeName, percentage)) + setValue = (availableBytes / capacityBytes) * 100.0 + nodePercentageGaugeVec.With(prometheus.Labels{"node_name": nodeName}).Set(setValue) + log.Debug().Msg(fmt.Sprintf("Node: %s percentage used: %f", nodeName, setValue)) } - return labelsList - } func setMetrics(node string) { - var labelsList []CollectMetric var data ephemeralStorageMetrics start := time.Now() @@ -480,31 +457,8 @@ func setMetrics(node string) { log.Warn().Msg(fmt.Sprintf("pod %s/%s on %s has no metrics on its ephemeral storage usage", podName, podNamespace, nodeName)) continue } - labelsList = append(labelsList, generateLabels(podName, podNamespace, nodeName, usedBytes, - availableBytes, capacityBytes, p.Volumes)...) - } - - // TODO: remove these lines - // add methods like podGaugeVec.With(x.labels).Set(x.value) to generate labels. - // and by first call partial remove on a per pod basis - // ex. podGaugeVec.DeletePartialMatch(prometheus.Labels{"pod_name": pod_name}) - for _, x := range labelsList { - switch x.name { - case "ephemeral_storage_pod_usage": - podGaugeVec.With(x.labels).Set(x.value) - case "ephemeral_storage_container_limit_percentage": - containerPercentageLimitsVec.With(x.labels).Set(x.value) - case "ephemeral_storage_container_volume_limit_percentage": - containerVolumePercentageLimitsVec.With(x.labels).Set(x.value) - case "ephemeral_storage_node_available": - nodeAvailableGaugeVec.With(x.labels).Set(x.value) - case "ephemeral_storage_node_capacity": - nodeCapacityGaugeVec.With(x.labels).Set(x.value) - case "ephemeral_storage_node_percentage": - nodePercentageGaugeVec.With(x.labels).Set(x.value) - - } - + createMetricsValues(podName, podNamespace, nodeName, usedBytes, + availableBytes, capacityBytes, p.Volumes) } adjustTime := sampleIntervalMill - time.Now().Sub(start).Milliseconds() @@ -519,113 +473,97 @@ func setMetrics(node string) { func createMetrics() { - // TODO: Will the metrics still show up if I also always register? - // Idea: I want to always registry so we can call clean up using a python like partials without conditionals. - if ephemeralStoragePodUsage { - podGaugeVec = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "ephemeral_storage_pod_usage", - Help: "Current ephemeral byte usage of pod", + podGaugeVec = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "ephemeral_storage_pod_usage", + Help: "Current ephemeral byte usage of pod", + }, + []string{ + // name of pod for Ephemeral Storage + "pod_name", + // namespace of pod for Ephemeral Storage + "pod_namespace", + // Name of Node where pod is placed. + "node_name", }, - []string{ - // name of pod for Ephemeral Storage - "pod_name", - // namespace of pod for Ephemeral Storage - "pod_namespace", - // Name of Node where pod is placed. - "node_name", - }, - ) - - prometheus.MustRegister(podGaugeVec) - - } - - if ephemeralStorageContainerLimitsPercentage { - containerPercentageLimitsVec = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "ephemeral_storage_container_limit_percentage", - Help: "Percentage of ephemeral storage used by a container in a pod", + ) + + prometheus.MustRegister(podGaugeVec) + + containerPercentageLimitsVec = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "ephemeral_storage_container_limit_percentage", + Help: "Percentage of ephemeral storage used by a container in a pod", + }, + []string{ + // name of pod for Ephemeral Storage + "pod_name", + // namespace of pod for Ephemeral Storage + "pod_namespace", + // Name of Node where pod is placed. + "node_name", + // Name of container + "container", }, - []string{ - // name of pod for Ephemeral Storage - "pod_name", - // namespace of pod for Ephemeral Storage - "pod_namespace", - // Name of Node where pod is placed. - "node_name", - // Name of container - "container", - }, - ) - - prometheus.MustRegister(containerPercentageLimitsVec) - - } - if ephemeralStorageContainerVolumeLimitsPercentage { - containerVolumePercentageLimitsVec = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "ephemeral_storage_container_volume_limit_percentage", - Help: "Percentage of ephemeral storage used by a container's volume in a pod", + ) + + prometheus.MustRegister(containerPercentageLimitsVec) + + containerPercentageVolumeLimitsVec = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "ephemeral_storage_container_volume_limit_percentage", + Help: "Percentage of ephemeral storage used by a container's volume in a pod", + }, + []string{ + // name of pod for Ephemeral Storage + "pod_name", + // namespace of pod for Ephemeral Storage + "pod_namespace", + // Name of Node where pod is placed. + "node_name", + // Name of container + "container", + // Name of Volume + "volume_name", + // Name of Mount Path + "mount_path", }, - []string{ - // name of pod for Ephemeral Storage - "pod_name", - // namespace of pod for Ephemeral Storage - "pod_namespace", - // Name of Node where pod is placed. - "node_name", - // Name of container - "container", - // Name of Volume - "volume_name", - // Name of Mount Path - "mount_path", - }, - ) + ) - prometheus.MustRegister(containerVolumePercentageLimitsVec) + prometheus.MustRegister(containerPercentageVolumeLimitsVec) - } - - if ephemeralStorageNodeAvailable { - nodeAvailableGaugeVec = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "ephemeral_storage_node_available", - Help: "Available ephemeral storage for a node", + nodeAvailableGaugeVec = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "ephemeral_storage_node_available", + Help: "Available ephemeral storage for a node", + }, + []string{ + // Name of Node where pod is placed. + "node_name", }, - []string{ - // Name of Node where pod is placed. - "node_name", - }, - ) + ) - prometheus.MustRegister(nodeAvailableGaugeVec) - } + prometheus.MustRegister(nodeAvailableGaugeVec) - if ephemeralStorageNodeCapacity { - nodeCapacityGaugeVec = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "ephemeral_storage_node_capacity", - Help: "Capacity of ephemeral storage for a node", + nodeCapacityGaugeVec = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "ephemeral_storage_node_capacity", + Help: "Capacity of ephemeral storage for a node", + }, + []string{ + // Name of Node where pod is placed. + "node_name", }, - []string{ - // Name of Node where pod is placed. - "node_name", - }, - ) + ) - prometheus.MustRegister(nodeCapacityGaugeVec) - } + prometheus.MustRegister(nodeCapacityGaugeVec) - if ephemeralStorageNodePercentage { - nodePercentageGaugeVec = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "ephemeral_storage_node_percentage", - Help: "Percentage of ephemeral storage used on a node", + nodePercentageGaugeVec = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "ephemeral_storage_node_percentage", + Help: "Percentage of ephemeral storage used on a node", + }, + []string{ + // Name of Node where pod is placed. + "node_name", }, - []string{ - // Name of Node where pod is placed. - "node_name", - }, - ) + ) - prometheus.MustRegister(nodePercentageGaugeVec) - } + prometheus.MustRegister(nodePercentageGaugeVec) if adjustedPollingRate { adjustedTimeGaugeVec = prometheus.NewGaugeVec(prometheus.GaugeOpts{