tempo-query: Make use of gRPC ReadinessProbe

Signed-off-by: Benedikt Bongartz <bongartz@klimlive.de>
grafana · Oct 15, 2024 · 5d6ecd6 · 5d6ecd6
1 parent 5bc672f
commit 5d6ecd6
Show file tree

Hide file tree

Showing 7 changed files with 42 additions and 6 deletions.
diff --git a/.chloggen/tempo-query_ReadinessProbe.yaml b/.chloggen/tempo-query_ReadinessProbe.yaml
@@ -0,0 +1,16 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: bug_fix
+
+# The name of the component, or a single word describing the area of concern, (e.g. tempostack, tempomonolithic, github action)
+component: tempostack
+
+# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Use the ReadinessProbe to better indicate when tempo-query is ready to accept requests. Improving the startup reliability by avoiding lost data.
+
+# One or more tracking issues related to the change
+issues: [1058]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: Without a readiness check in place, there is a risk that data will be lost when the queryfrontend pod is ready but the tempo query API is not yet available.
diff --git a/Makefile b/Makefile
@@ -2,7 +2,7 @@
 OPERATOR_VERSION ?= 0.13.0
 TEMPO_VERSION ?= 2.6.0
 JAEGER_QUERY_VERSION ?= 1.62.0
-TEMPO_QUERY_VERSION ?= main-a2f70c9
+TEMPO_QUERY_VERSION ?= main-4726342
 TEMPO_GATEWAY_VERSION ?= main-2024-08-05-11d0d94
 TEMPO_GATEWAY_OPA_VERSION ?= main-2024-04-29-914c13f
 OAUTH_PROXY_VERSION=4.14

diff --git a/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml b/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml
@@ -1453,7 +1453,7 @@ spec:
                 - name: RELATED_IMAGE_JAEGER_QUERY
                   value: docker.io/jaegertracing/jaeger-query:1.62.0
                 - name: RELATED_IMAGE_TEMPO_QUERY
-                  value: docker.io/grafana/tempo-query:main-a2f70c9
+                  value: docker.io/grafana/tempo-query:main-4726342
                 - name: RELATED_IMAGE_TEMPO_GATEWAY
                   value: quay.io/observatorium/api:main-2024-08-05-11d0d94
                 - name: RELATED_IMAGE_TEMPO_GATEWAY_OPA
@@ -1602,7 +1602,7 @@ spec:
     name: tempo
   - image: docker.io/jaegertracing/jaeger-query:1.62.0
     name: jaeger-query
-  - image: docker.io/grafana/tempo-query:main-a2f70c9
+  - image: docker.io/grafana/tempo-query:main-4726342
     name: tempo-query
   - image: quay.io/observatorium/api:main-2024-08-05-11d0d94
     name: tempo-gateway

diff --git a/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml b/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml
@@ -1463,7 +1463,7 @@ spec:
                 - name: RELATED_IMAGE_JAEGER_QUERY
                   value: docker.io/jaegertracing/jaeger-query:1.62.0
                 - name: RELATED_IMAGE_TEMPO_QUERY
-                  value: docker.io/grafana/tempo-query:main-a2f70c9
+                  value: docker.io/grafana/tempo-query:main-4726342
                 - name: RELATED_IMAGE_TEMPO_GATEWAY
                   value: quay.io/observatorium/api:main-2024-08-05-11d0d94
                 - name: RELATED_IMAGE_TEMPO_GATEWAY_OPA
@@ -1623,7 +1623,7 @@ spec:
     name: tempo
   - image: docker.io/jaegertracing/jaeger-query:1.62.0
     name: jaeger-query
-  - image: docker.io/grafana/tempo-query:main-a2f70c9
+  - image: docker.io/grafana/tempo-query:main-4726342
     name: tempo-query
   - image: quay.io/observatorium/api:main-2024-08-05-11d0d94
     name: tempo-gateway

diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml
@@ -44,7 +44,7 @@ spec:
         - name: RELATED_IMAGE_JAEGER_QUERY
           value: docker.io/jaegertracing/jaeger-query:1.62.0
         - name: RELATED_IMAGE_TEMPO_QUERY
-          value: docker.io/grafana/tempo-query:main-a2f70c9
+          value: docker.io/grafana/tempo-query:main-4726342
         - name: RELATED_IMAGE_TEMPO_GATEWAY
           value: quay.io/observatorium/api:main-2024-08-05-11d0d94
         - name: RELATED_IMAGE_TEMPO_GATEWAY_OPA

diff --git a/internal/manifests/queryfrontend/query_frontend.go b/internal/manifests/queryfrontend/query_frontend.go
@@ -311,6 +311,16 @@ func deployment(params manifestutils.Params) (*appsv1.Deployment, error) {
 			},
 			Resources:       tempoQueryResources(tempo),
 			SecurityContext: manifestutils.TempoContainerSecurityContext(),
+			ReadinessProbe: &corev1.Probe{
+				ProbeHandler: corev1.ProbeHandler{
+					GRPC: &corev1.GRPCAction{
+						Port: manifestutils.PortTempoGRPCQuery,
+					},
+				},
+				TimeoutSeconds:   1,
+				PeriodSeconds:    5,
+				FailureThreshold: 12,
+			},
 		}
 		jaegerQueryVolume := corev1.Volume{
 			Name: manifestutils.TmpStorageVolumeName + "-query",

diff --git a/internal/manifests/queryfrontend/query_frontend_test.go b/internal/manifests/queryfrontend/query_frontend_test.go
@@ -289,6 +289,16 @@ func getExpectedDeployment(withJaeger bool) *v1.Deployment {
 					ReadOnly:  true,
 				},
 			},
+			ReadinessProbe: &corev1.Probe{
+				ProbeHandler: corev1.ProbeHandler{
+					GRPC: &corev1.GRPCAction{
+						Port: manifestutils.PortTempoGRPCQuery,
+					},
+				},
+				TimeoutSeconds:   1,
+				PeriodSeconds:    5,
+				FailureThreshold: 12,
+			},
 			Resources: corev1.ResourceRequirements{
 				Limits: corev1.ResourceList{
 					corev1.ResourceCPU:    *resource.NewMilliQuantity(45, resource.BinarySI),