feat: add metrics for Prometheus (#29)

README.md

@@ -103,7 +103,7 @@ this project.
 
 ## Acknowledgments
 
-The [Spegel] project has greatly inspired this work, and a big THANK YOU to Philip Laine and Simon Gottschlag at Xenit
+The [Spegel] project has inspired this work; thanks to Philip Laine and Simon Gottschlag at Xenit
 for generously sharing their insights with us. A hat tip also to the [DADI P2P Proxy] project for demonstrating the
 integration with [Overlaybd].
 

build/ci/scripts/azure.sh

@@ -164,6 +164,7 @@ print_peerd_metrics() {
     for pod in $( echo "$p" | tr -s " " "\012" ); do
         echo "checking pod '$pod' for metrics"
         kubectl -n peerd-ns exec -i $pod -- bash -c "cat /var/log/peerdmetrics"
+        kubectl --context=$KIND_CLUSTER_CONTEXT -n peerd-ns exec -i $pod -- bash -c "curl http://localhost:5004/metrics/prometheus"
     done
 }
 

build/ci/scripts/kind.sh

@@ -124,6 +124,7 @@ print_p2p_metrics() {
     for pod in $( echo "$p" | tr -s " " "\012" ); do
         echo "checking pod '$pod' for metrics"
         kubectl --context=$KIND_CLUSTER_CONTEXT -n peerd-ns exec -i $pod -- bash -c "cat /var/log/peerdmetrics"
+        kubectl --context=$KIND_CLUSTER_CONTEXT -n peerd-ns exec -i $pod -- bash -c "curl http://localhost:5004/metrics/prometheus"
     done
 }
 

build/package/peerd-helm/Chart.yaml

@@ -15,10 +15,10 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.0.1-alpha
+version: 0.0.2-alpha
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "0.0.1-alpha"
+appVersion: "0.0.4-alpha"

build/package/peerd-helm/templates/_helpers.tpl

@@ -0,0 +1,60 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "peerd.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{- define "peerd.namespace" -}}
+{{ include "peerd.name" . }}-ns
+{{- end }}
+
+{{- define "peerd.serviceAccountName" -}}
+{{ include "peerd.name" . }}-sa
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "peerd.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "peerd.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "peerd.labels" -}}
+helm.sh/chart: {{ include "peerd.chart" . }}
+{{ include "peerd.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Common selector labels
+*/}}
+{{- define "peerd.selectorLabels" -}}
+app: {{ include "peerd.name" . }}
+app.kubernetes.io/name: {{ include "peerd.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}

build/package/peerd-helm/templates/app.yml

@@ -1,15 +1,45 @@
 apiVersion: v1
 kind: Namespace
 metadata:
-  name: peerd-ns
+  name: {{ include "peerd.namespace" . }}
+  labels:
+    {{- include "peerd.labels" . | nindent 4 }}
+---
+kind: ConfigMap
+apiVersion: v1
+data:
+  prometheus-config: |-
+    global:
+      scrape_interval: 15s
+    scrape_configs:
+    - job_name: peerd
+      kubernetes_sd_configs:
+        - role: pod
+      relabel_configs:
+        - source_labels: [__meta_kubernetes_pod_label_app]
+          action: keep
+          regex: peerd
+        - source_labels: [__meta_kubernetes_pod_container_port_number]
+          action: keep
+          regex: "5004"
+      metrics_path: /metrics/prometheus
+metadata:
+  name: ama-metrics-prometheus-config
+  namespace: kube-system
+  labels:
+    {{- include "peerd.labels" . | nindent 4 }}
 ---
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
-  name: &name peerd
-  namespace: peerd-ns
+  name: &name {{ include "peerd.name" . }}
+  namespace: {{ include "peerd.namespace" . }}
   labels:
-    app: *name
+    {{- include "peerd.labels" . | nindent 4 }}
+  annotations:
+    prometheus.io/scrape: 'true'
+    prometheus.io/path: '/metrics/prometheus'
+    prometheus.io/port: '5004'
 spec:
   selector:
     matchLabels:
@@ -19,16 +49,7 @@ spec:
       labels:
         app: *name
     spec:
-      # affinity:
-      #   nodeAffinity:
-      #     requiredDuringSchedulingIgnoredDuringExecution:
-      #       nodeSelectorTerms:
-      #         - matchExpressions:
-      #             - key: p2p-nodepool
-      #               operator: In
-      #               values:
-      #                 - "true"
-      serviceAccountName: peerd-sa
+      serviceAccountName: {{ include "peerd.serviceAccountName" . }}
       containers:
         - image: "{{ .Values.peerd.image.ref }}"
           imagePullPolicy: "{{ .Values.peerd.image.pullPolicy }}"
@@ -50,6 +71,10 @@ spec:
               name: http
             - containerPort: 5001
               name: https
+            - containerPort: 5003
+              name: router
+            - containerPort: 5004
+              name: metrics
           volumeMounts:
             - name: metricsmount
               mountPath: "/var/log/peerdmetrics"
@@ -74,10 +99,14 @@ spec:
 apiVersion: v1
 kind: Service
 metadata:
-  name: &name peerd
-  namespace: peerd-ns
+  name: &name {{ include "peerd.name" . }}
+  namespace: {{ include "peerd.namespace" . }}
   labels:
-    app: *name
+    {{- include "peerd.labels" . | nindent 4 }}
+  annotations:
+    prometheus.io/scrape: 'true'
+    prometheus.io/path: '/metrics/prometheus'
+    prometheus.io/port: '30004'
 spec:
   type: NodePort
   selector:
@@ -93,3 +122,8 @@ spec:
       port: 5001
       nodePort: 30001
       targetPort: https
+    - name: metrics
+      protocol: TCP
+      port: 5004
+      nodePort: 30004
+      targetPort: metrics

build/package/peerd-helm/templates/rbac.yml

@@ -1,29 +1,29 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: peerd-sa
-  namespace: peerd-ns
+  name: {{ include "peerd.serviceAccountName" . }}
+  namespace: {{ include "peerd.namespace" . }}
   labels:
-    app: peerd
+    {{- include "peerd.labels" . | nindent 4 }}
 ---
 apiVersion: v1
 kind: Secret
 metadata:
-  name: peerd-sa-secret
-  namespace: peerd-ns
+  name: {{ include "peerd.serviceAccountName" . }}-secret
+  namespace: {{ include "peerd.namespace" . }}
   labels:
-    app: peerd
+    {{- include "peerd.labels" . | nindent 4 }}
   annotations:
-    kubernetes.io/service-account.name: peerd-sa
+    kubernetes.io/service-account.name: {{ include "peerd.serviceAccountName" . }}
 type: kubernetes.io/service-account-token
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
-  name: peerd-manager-role
-  namespace: peerd-ns
+  name: {{ include "peerd.name" . }}-manager-role
+  namespace: {{ include "peerd.namespace" . }}
   labels:
-    app: peerd
+    {{- include "peerd.labels" . | nindent 4 }}
 rules:
   - apiGroups: ["coordination.k8s.io"]
     resources: ["configmaps"]
@@ -38,23 +38,25 @@ rules:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
-  name: peerd-manager-role-binding
-  namespace: peerd-ns
+  name: {{ include "peerd.name" . }}-manager-role-binding
+  namespace: {{ include "peerd.namespace" . }}
   labels:
-    app: peerd
+    {{- include "peerd.labels" . | nindent 4 }}
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: Role
   name: peerd-manager-role
 subjects:
   - kind: ServiceAccount
-    name: peerd-sa
-    namespace: peerd-ns
+    name: {{ include "peerd.serviceAccountName" . }}
+    namespace: {{ include "peerd.namespace" . }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
   name: node-reader-and-event-creator-clusterrole
+  labels:
+    {{- include "peerd.labels" . | nindent 4 }}
 rules:
 - apiGroups: [""]
   resources: ["nodes"]
@@ -67,10 +69,12 @@ apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
   name: node-reader-and-event-creator-clusterrole-binding
+  labels:
+    {{- include "peerd.labels" . | nindent 4 }}
 subjects:
 - kind: ServiceAccount
-  name: peerd-sa
-  namespace: peerd-ns
+  name: {{ include "peerd.serviceAccountName" . }}
+  namespace: {{ include "peerd.namespace" . }}
 roleRef:
   kind: ClusterRole
   name: node-reader-and-event-creator-clusterrole

cmd/proxy/cmd.go

@@ -6,6 +6,7 @@ type ServerCmd struct {
 	HttpAddr        string `arg:"--http-addr" help:"address of the server" default:"127.0.0.1:5000"`
 	HttpsAddr       string `arg:"--https-addr" help:"address of the server" default:"0.0.0.0:5001"`
 	RouterAddr      string `arg:"--router-addr" help:"address of the router (p2p)" default:"0.0.0.0:5003"`
+	PromAddr        string `arg:"--prom-addr" help:"address of prometheus metrics endpoint" default:"0.0.0.0:5004"`
 	PrefetchWorkers int    `arg:"--prefetch-workers" help:"number of workers to prefetch content" default:"50"`
 
 	// Mirror configuration.

cmd/proxy/main.go

@@ -24,6 +24,7 @@ import (
 	"github.com/azure/peerd/internal/state"
 	"github.com/azure/peerd/pkg/containerd"
 	"github.com/azure/peerd/pkg/k8s"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/rs/zerolog"
 	"github.com/spf13/afero"
 	"golang.org/x/sync/errgroup"
@@ -164,41 +165,45 @@ func serverCommand(ctx context.Context, args *ServerCmd) (err error) {
 		Handler:   handler,
 		TLSConfig: r.Net().DefaultTLSConfig(),
 	}
-
 	g.Go(func() error {
 		if err := httpsSrv.ListenAndServeTLS("", ""); err != nil && !errors.Is(err, http.ErrServerClosed) {
 			return err
 		}
 		return nil
 	})
+	g.Go(func() error {
+		<-ctx.Done()
+		shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cancel()
+		return httpsSrv.Shutdown(shutdownCtx)
+	})
 
 	httpSrv := &http.Server{
 		Addr:    args.HttpAddr,
 		Handler: handler,
 	}
-
 	g.Go(func() error {
 		if err := httpSrv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
 			return err
 		}
 		return nil
 	})
-
 	g.Go(func() error {
 		<-ctx.Done()
 		shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 		defer cancel()
-		return httpsSrv.Shutdown(shutdownCtx)
+		return httpSrv.Shutdown(shutdownCtx)
 	})
 
 	g.Go(func() error {
-		<-ctx.Done()
-		shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
-		defer cancel()
-		return httpSrv.Shutdown(shutdownCtx)
+		http.Handle("/metrics/prometheus", promhttp.Handler())
+		if err = http.ListenAndServe(args.PromAddr, nil); err != nil && !errors.Is(err, http.ErrServerClosed) {
+			return err
+		}
+		return nil
 	})
 
-	l.Info().Str("https", args.HttpsAddr).Str("http", args.HttpAddr).Msg("server start")
+	l.Info().Str("https", args.HttpsAddr).Str("http", args.HttpAddr).Str("router", args.RouterAddr).Str("prom", args.PromAddr).Msg("server start")
 	err = g.Wait()
 	if err != nil {
 		return err

go.mod

@@ -14,6 +14,7 @@ require (
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.1.0
 	github.com/pelletier/go-toml/v2 v2.1.1
+	github.com/prometheus/client_golang v1.18.0
 	github.com/rs/zerolog v1.32.0
 	github.com/schollz/progressbar/v3 v3.14.2
 	github.com/swaggo/swag v1.16.3
@@ -130,7 +131,6 @@ require (
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/polydawn/refmt v0.89.0 // indirect
-	github.com/prometheus/client_golang v1.18.0 // indirect
 	github.com/prometheus/client_model v0.6.0 // indirect
 	github.com/prometheus/common v0.47.0 // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect