[VSSL-7955] Add prometheus-adapter (#41)

* Add prometheus-adapter * Update Chart dependency
vessl-ai · Feb 27, 2024 · ddbd25b · ddbd25b
1 parent 0a3b202
commit ddbd25b
Show file tree

Hide file tree

Showing 3 changed files with 61 additions and 2 deletions.
diff --git a/charts/vessl/Chart.lock b/charts/vessl/Chart.lock
@@ -29,5 +29,8 @@ dependencies:
 - name: harbor
   repository: ""
   version: 1.0.1
-digest: sha256:101f07222ecd1b841f3c7a1b123a4aa6b3e9d06d1a08cf2bc087558f36cfe247
-generated: "2023-11-21T17:30:52.04519+09:00"
+- name: prometheus-adapter
+  repository: https://prometheus-community.github.io/helm-charts
+  version: 4.9.0
+digest: sha256:1d6fedefba703d9c2fbf32e4e4a695a9598066ab1b25701f18ed6b8f86f278c6
+generated: "2024-02-27T12:05:24.229959+09:00"
diff --git a/charts/vessl/Chart.yaml b/charts/vessl/Chart.yaml
@@ -46,3 +46,7 @@ dependencies:
 - name: harbor
   version: "1.0.1"
   condition: harbor.enabled
+- name: prometheus-adapter
+  version: "4.9.0"
+  condition: prometheus-adapter.enabled
+  repository: https://prometheus-community.github.io/helm-charts
diff --git a/charts/vessl/values.yaml b/charts/vessl/values.yaml
@@ -393,3 +393,55 @@ longhorn:
 # https://github.com/vessl-ai/helm-charts/blob/main/charts/vessl/charts/harbor/values.yaml
 harbor:
   enabled: false
+
+# https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-adapter
+prometheus-adapter:
+  enabled: false
+
+  tolerations:
+    - key: "node-role.kubernetes.io/master"
+      value: ""
+      effect: "NoSchedule"
+    - key: "node-role.kubernetes.io/control-plane"
+      operator: "Exists"
+      effect: "NoSchedule"
+    - key: "v1.k8s.vessl.ai/dedicated"
+      operator: "Exists"
+      effect: "NoSchedule"
+
+  affinity:
+    nodeAffinity:
+      preferredDuringSchedulingIgnoredDuringExecution:
+        - weight: 1
+          preference:
+            matchExpressions:
+              - key: "node-role.kubernetes.io/master"
+                operator: "In"
+                values: [""]
+        - weight: 1
+          preference:
+            matchExpressions:
+              - key: "node-role.kubernetes.io/control-plane"
+                operator: "Exists"
+        - weight: 2
+          preference:
+            matchExpressions:
+              - key: "v1.k8s.vessl.ai/dedicated"
+                operator: "In"
+                values: ["manager"]
+
+  prometheus:
+    url: http://vessl-prometheus-remote-write-server.vessl.svc
+
+  rules:
+    default: false
+
+    custom:
+      - seriesQuery: 'DCGM_FI_DEV_GPU_UTIL{pod!=""}'
+        resources:
+          overrides:
+            namespace: {resource: "namespace"}
+            pod: {resource: "pod"}
+        name:
+          as: "gpu_util"
+        metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>)