feat: add headlessService for grafana high availability

grafana · Jan 30, 2025 · ed971a5 · ed971a5
1 parent ff3fb07
commit ed971a5
Showing 7 changed files with 92 additions and 0 deletions.
diff --git a/controllers/config/operator_constants.go b/controllers/config/operator_constants.go
@@ -22,6 +22,8 @@ const (
 	GrafanaHttpPort       int = 3000
 	GrafanaHttpPortName       = "grafana"
 	GrafanaServerProtocol     = "http"
+	GrafanaAlertPort      int = 9094
+	GrafanaAlertPortName      = "grafana-alert"
 
 	// Data storage
 	GrafanaProvisionPluginVolumeName    = "grafana-provision-plugins"

diff --git a/controllers/model/grafana_resources.go b/controllers/model/grafana_resources.go
@@ -81,6 +81,18 @@ func GetGrafanaService(cr *grafanav1beta1.Grafana, scheme *runtime.Scheme) *v1.S
 	return service
 }
 
+func GetGrafanaHeadlessService(cr *grafanav1beta1.Grafana, scheme *runtime.Scheme) *v1.Service {
+	service := &v1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      fmt.Sprintf("%s-alerting", cr.Name),
+			Namespace: cr.Namespace,
+			Labels:    CommonLabels,
+		},
+	}
+	controllerutil.SetControllerReference(cr, service, scheme) //nolint:errcheck
+	return service
+}
+
 func GetGrafanaIngress(cr *grafanav1beta1.Grafana, scheme *runtime.Scheme) *v12.Ingress {
 	ingress := &v12.Ingress{
 		ObjectMeta: metav1.ObjectMeta{

diff --git a/controllers/reconcilers/grafana/deployment_reconciler.go b/controllers/reconcilers/grafana/deployment_reconciler.go
@@ -6,6 +6,7 @@ import (
 	"strings"
 
 	"github.com/grafana/grafana-operator/v5/api/v1beta1"
+	"github.com/grafana/grafana-operator/v5/controllers/config"
 	config2 "github.com/grafana/grafana-operator/v5/controllers/config"
 	"github.com/grafana/grafana-operator/v5/controllers/model"
 	"github.com/grafana/grafana-operator/v5/controllers/reconcilers"
@@ -184,6 +185,16 @@ func getContainers(cr *v1beta1.Grafana, scheme *runtime.Scheme, vars *v1beta1.Op
 		Value: config2.GrafanaDataPath,
 	})
 
+	// env var to get Pod IP from downward API for gossip (useful for unified alerting).
+	envVars = append(envVars, v1.EnvVar{
+		Name: "POD_IP",
+		ValueFrom: &v1.EnvVarSource{
+			FieldRef: &v1.ObjectFieldSelector{
+				FieldPath: "status.podIP",
+			},
+		},
+	})
+
 	containers = append(containers, v1.Container{
 		Name:       "grafana",
 		Image:      image,
@@ -195,6 +206,11 @@ func getContainers(cr *v1beta1.Grafana, scheme *runtime.Scheme, vars *v1beta1.Op
 				ContainerPort: int32(GetGrafanaPort(cr)), // #nosec G115
 				Protocol:      "TCP",
 			},
+			{
+				Name:          config.GrafanaAlertPortName,
+				ContainerPort: int32(config.GrafanaAlertPort),
+				Protocol:      "TCP",
+			},
 		},
 		Env:                      envVars,
 		Resources:                getResources(),

diff --git a/controllers/reconcilers/grafana/grafana_service_reconciler.go b/controllers/reconcilers/grafana/grafana_service_reconciler.go
@@ -54,6 +54,24 @@ func (r *ServiceReconciler) Reconcile(ctx context.Context, cr *v1beta1.Grafana,
 			int32(GetGrafanaPort(cr))) // #nosec G115
 	}
 
+	// Headless service for grafana unified alerting
+	headlessService := model.GetGrafanaHeadlessService(cr, scheme)
+	_, err = controllerutil.CreateOrUpdate(ctx, r.client, headlessService, func() error {
+		model.SetCommonLabels(service)
+		service.Spec = v1.ServiceSpec{
+			ClusterIP: "None",
+			Ports:     getHeadlessServicePorts(cr),
+			Selector: map[string]string{
+				"app": cr.Name,
+			},
+			Type: v1.ServiceTypeClusterIP,
+		}
+		return nil
+	})
+	if err != nil {
+		return v1beta1.OperatorStageResultFailed, err
+	}
+
 	return v1beta1.OperatorStageResultSuccess, nil
 }
 
@@ -95,3 +113,18 @@ func getServicePorts(cr *v1beta1.Grafana) []v1.ServicePort {
 
 	return defaultPorts
 }
+
+func getHeadlessServicePorts(_ *v1beta1.Grafana) []v1.ServicePort {
+	intPort := int32(config.GrafanaAlertPort)
+
+	defaultPorts := []v1.ServicePort{
+		{
+			Name:       config.GrafanaAlertPortName,
+			Protocol:   "TCP",
+			Port:       intPort,
+			TargetPort: intstr.FromInt32(intPort),
+		},
+	}
+
+	return defaultPorts
+}
diff --git a/examples/multiple_replicas/resources.yaml b/examples/multiple_replicas/resources.yaml
@@ -85,3 +85,12 @@ spec:
       name: "grafana"
       user: "grafana"
       password: "grafana"
+    # Configure HA for Grafana alerting
+    # https://grafana.com/docs/grafana/latest/alerting/set-up/configure-high-availability/
+    unified_alerting:
+      enabled: true
+      ha_listen_address: "${POD_IP}:9094"
+      ha_peers: "grafana-alerting:9094"
+      ha_advertise_address: "${POD_IP}:9094"
+      ha_peer_timeout: 15s
+      ha_reconnect_timeout: 2m
diff --git a/tests/e2e/examples/basic/assertions.yaml b/tests/e2e/examples/basic/assertions.yaml
@@ -19,6 +19,16 @@ metadata:
 spec: {}
 ---
 apiVersion: v1
+kind: Service
+metadata:
+  name: grafana-alerting
+  ownerReferences:
+  - apiVersion: grafana.integreatly.org/v1beta1
+    kind: Grafana
+    name: grafana
+spec: {}
+---
+apiVersion: v1
 kind: ConfigMap
 metadata:
   name: grafana-ini

diff --git a/tests/e2e/examples/crossnamespace/assertions.yaml b/tests/e2e/examples/crossnamespace/assertions.yaml
@@ -19,6 +19,16 @@ metadata:
 spec: {}
 ---
 apiVersion: v1
+kind: Service
+metadata:
+  name: grafana-alerting
+  ownerReferences:
+  - apiVersion: grafana.integreatly.org/v1beta1
+    kind: Grafana
+    name: grafana
+spec: {}
+---
+apiVersion: v1
 kind: ConfigMap
 metadata:
   name: grafana-ini