From 78b5cea6fcdf76f2ff1878582745cf4dfd12d6fa Mon Sep 17 00:00:00 2001 From: Israel Blancas Date: Thu, 3 Oct 2024 11:52:42 +0200 Subject: [PATCH 01/10] [chore] Improve replication factor documentation (#1042) * Improve replication factor documentation Signed-off-by: Israel Blancas * Update apis/tempo/v1alpha1/tempostack_types.go Co-authored-by: Andreas Gerstmayr * Apply changes requested in CR Signed-off-by: Israel Blancas --------- Signed-off-by: Israel Blancas Co-authored-by: Andreas Gerstmayr --- apis/tempo/v1alpha1/tempostack_types.go | 2 +- .../manifests/tempo-operator.clusterserviceversion.yaml | 7 ++++--- .../community/manifests/tempo.grafana.com_tempostacks.yaml | 5 +++-- .../manifests/tempo-operator.clusterserviceversion.yaml | 7 ++++--- .../openshift/manifests/tempo.grafana.com_tempostacks.yaml | 5 +++-- config/crd/bases/tempo.grafana.com_tempostacks.yaml | 5 +++-- .../bases/tempo-operator.clusterserviceversion.yaml | 5 +++-- .../bases/tempo-operator.clusterserviceversion.yaml | 5 +++-- docs/spec/tempo.grafana.com_tempostacks.yaml | 2 +- 9 files changed, 25 insertions(+), 18 deletions(-) diff --git a/apis/tempo/v1alpha1/tempostack_types.go b/apis/tempo/v1alpha1/tempostack_types.go index c42fbb536..447b93fee 100644 --- a/apis/tempo/v1alpha1/tempostack_types.go +++ b/apis/tempo/v1alpha1/tempostack_types.go @@ -103,7 +103,7 @@ type TempoStackSpec struct { // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Tempo Component Templates" Template TempoTemplateSpec `json:"template,omitempty"` - // ReplicationFactor is used to define how many component replicas should exist. + // The replication factor is a configuration setting that determines how many ingesters need to acknowledge the data from the distributors before accepting a span. // // +optional // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Replication Factor" diff --git a/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml b/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml index fade09105..bd74cacaf 100644 --- a/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml +++ b/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml @@ -74,7 +74,7 @@ metadata: capabilities: Deep Insights categories: Logging & Tracing,Monitoring containerImage: ghcr.io/grafana/tempo-operator/tempo-operator:v0.13.0 - createdAt: "2024-09-21T07:00:19Z" + createdAt: "2024-10-03T09:33:24Z" description: Create and manage deployments of Tempo, a high-scale distributed tracing backend. operatorframework.io/cluster-monitoring: "true" @@ -718,8 +718,9 @@ spec: 0 to 1. displayName: Sampling Fraction path: observability.tracing.sampling_fraction - - description: ReplicationFactor is used to define how many component replicas - should exist. + - description: The replication factor is a configuration setting that determines + how many ingesters need to acknowledge the data from the distributors before + accepting a span. displayName: Replication Factor path: replicationFactor - description: Resources defines resources configuration. diff --git a/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml b/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml index 3d4a67273..27f123dca 100644 --- a/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml +++ b/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml @@ -293,8 +293,9 @@ spec: type: object type: object replicationFactor: - description: ReplicationFactor is used to define how many component - replicas should exist. + description: The replication factor is a configuration setting that + determines how many ingesters need to acknowledge the data from + the distributors before accepting a span. type: integer resources: description: Resources defines resources configuration. diff --git a/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml b/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml index 7d7c22021..9406e26b4 100644 --- a/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml +++ b/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml @@ -74,7 +74,7 @@ metadata: capabilities: Deep Insights categories: Logging & Tracing,Monitoring containerImage: ghcr.io/grafana/tempo-operator/tempo-operator:v0.13.0 - createdAt: "2024-09-21T07:00:18Z" + createdAt: "2024-10-03T09:33:23Z" description: Create and manage deployments of Tempo, a high-scale distributed tracing backend. operatorframework.io/cluster-monitoring: "true" @@ -718,8 +718,9 @@ spec: 0 to 1. displayName: Sampling Fraction path: observability.tracing.sampling_fraction - - description: ReplicationFactor is used to define how many component replicas - should exist. + - description: The replication factor is a configuration setting that determines + how many ingesters need to acknowledge the data from the distributors before + accepting a span. displayName: Replication Factor path: replicationFactor - description: Resources defines resources configuration. diff --git a/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml b/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml index 3d4a67273..27f123dca 100644 --- a/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml +++ b/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml @@ -293,8 +293,9 @@ spec: type: object type: object replicationFactor: - description: ReplicationFactor is used to define how many component - replicas should exist. + description: The replication factor is a configuration setting that + determines how many ingesters need to acknowledge the data from + the distributors before accepting a span. type: integer resources: description: Resources defines resources configuration. diff --git a/config/crd/bases/tempo.grafana.com_tempostacks.yaml b/config/crd/bases/tempo.grafana.com_tempostacks.yaml index 264b9cd3e..2cdfc2c3b 100644 --- a/config/crd/bases/tempo.grafana.com_tempostacks.yaml +++ b/config/crd/bases/tempo.grafana.com_tempostacks.yaml @@ -289,8 +289,9 @@ spec: type: object type: object replicationFactor: - description: ReplicationFactor is used to define how many component - replicas should exist. + description: The replication factor is a configuration setting that + determines how many ingesters need to acknowledge the data from + the distributors before accepting a span. type: integer resources: description: Resources defines resources configuration. diff --git a/config/manifests/community/bases/tempo-operator.clusterserviceversion.yaml b/config/manifests/community/bases/tempo-operator.clusterserviceversion.yaml index 3d85811b5..419a32f01 100644 --- a/config/manifests/community/bases/tempo-operator.clusterserviceversion.yaml +++ b/config/manifests/community/bases/tempo-operator.clusterserviceversion.yaml @@ -647,8 +647,9 @@ spec: 0 to 1. displayName: Sampling Fraction path: observability.tracing.sampling_fraction - - description: ReplicationFactor is used to define how many component replicas - should exist. + - description: The replication factor is a configuration setting that determines + how many ingesters need to acknowledge the data from the distributors before + accepting a span. displayName: Replication Factor path: replicationFactor - description: Resources defines resources configuration. diff --git a/config/manifests/openshift/bases/tempo-operator.clusterserviceversion.yaml b/config/manifests/openshift/bases/tempo-operator.clusterserviceversion.yaml index 9a1fda20f..1b04fa519 100644 --- a/config/manifests/openshift/bases/tempo-operator.clusterserviceversion.yaml +++ b/config/manifests/openshift/bases/tempo-operator.clusterserviceversion.yaml @@ -647,8 +647,9 @@ spec: 0 to 1. displayName: Sampling Fraction path: observability.tracing.sampling_fraction - - description: ReplicationFactor is used to define how many component replicas - should exist. + - description: The replication factor is a configuration setting that determines + how many ingesters need to acknowledge the data from the distributors before + accepting a span. displayName: Replication Factor path: replicationFactor - description: Resources defines resources configuration. diff --git a/docs/spec/tempo.grafana.com_tempostacks.yaml b/docs/spec/tempo.grafana.com_tempostacks.yaml index 3693381ad..4ac207289 100644 --- a/docs/spec/tempo.grafana.com_tempostacks.yaml +++ b/docs/spec/tempo.grafana.com_tempostacks.yaml @@ -54,7 +54,7 @@ spec: # TempoStackSpec defines the desired st tracing: # Tracing defines a config for operands. jaeger_agent_endpoint: "localhost:6831" # JaegerAgentEndpoint defines the jaeger endpoint data gets send to. sampling_fraction: "" # SamplingFraction defines the sampling ratio. Valid values are 0 to 1. - replicationFactor: 0 # ReplicationFactor is used to define how many component replicas should exist. + replicationFactor: 0 # The replication factor is a configuration setting that determines how many ingesters need to acknowledge the data from the distributors before accepting a span. retention: # Retention period defined by dataset. User can specify how long data should be stored. global: # Global is used to configure global retention. traces: "" # Traces defines retention period. Supported parameter suffixes are "s", "m" and "h". example: 336h default: value is 48h. From e2ae4c0e5614095944403bd161920a023f05e82b Mon Sep 17 00:00:00 2001 From: Pavol Loffay Date: Fri, 4 Oct 2024 16:14:28 +0200 Subject: [PATCH 02/10] Fix OCP e2e tempo-query multitenant tests (#1047) Signed-off-by: Pavol Loffay --- Makefile | 5 +++-- .../manifests/tempo-operator.clusterserviceversion.yaml | 6 +++--- .../manifests/tempo-operator.clusterserviceversion.yaml | 6 +++--- config/manager/manager.yaml | 2 +- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 81511b88e..fe152e9c0 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,8 @@ OPERATOR_VERSION ?= 0.13.0 TEMPO_VERSION ?= 2.5.0 TEMPO_QUERY_VERSION ?= main-2999520 -JAEGER_QUERY_VERSION ?= 1.60 +# TODO change this to a release version. This has https://github.com/jaegertracing/jaeger/commit/d6631f5f2370cfc3a49efce312491031fb387600 +JAEGER_QUERY_VERSION ?= d6631f5f2370cfc3a49efce312491031fb387600 TEMPO_GATEWAY_VERSION ?= main-2024-08-05-11d0d94 TEMPO_GATEWAY_OPA_VERSION ?= main-2024-04-29-914c13f OAUTH_PROXY_VERSION=4.12 @@ -11,7 +12,7 @@ MIN_KUBERNETES_VERSION ?= 1.25.0 MIN_OPENSHIFT_VERSION ?= 4.12 TEMPO_IMAGE ?= docker.io/grafana/tempo:$(TEMPO_VERSION) -JAEGER_QUERY_IMAGE ?= docker.io/jaegertracing/jaeger-query:$(JAEGER_QUERY_VERSION) +JAEGER_QUERY_IMAGE ?= docker.io/jaegertracing/jaeger-query-snapshot:$(JAEGER_QUERY_VERSION) TEMPO_QUERY_IMAGE ?= docker.io/grafana/tempo-query:$(TEMPO_QUERY_VERSION) TEMPO_GATEWAY_IMAGE ?= quay.io/observatorium/api:$(TEMPO_GATEWAY_VERSION) TEMPO_GATEWAY_OPA_IMAGE ?= quay.io/observatorium/opa-openshift:$(TEMPO_GATEWAY_OPA_VERSION) diff --git a/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml b/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml index bd74cacaf..b623f6b16 100644 --- a/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml +++ b/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml @@ -74,7 +74,7 @@ metadata: capabilities: Deep Insights categories: Logging & Tracing,Monitoring containerImage: ghcr.io/grafana/tempo-operator/tempo-operator:v0.13.0 - createdAt: "2024-10-03T09:33:24Z" + createdAt: "2024-10-04T12:51:28Z" description: Create and manage deployments of Tempo, a high-scale distributed tracing backend. operatorframework.io/cluster-monitoring: "true" @@ -1425,7 +1425,7 @@ spec: - name: RELATED_IMAGE_TEMPO value: docker.io/grafana/tempo:2.5.0 - name: RELATED_IMAGE_JAEGER_QUERY - value: docker.io/jaegertracing/jaeger-query:1.60 + value: docker.io/jaegertracing/jaeger-query-snapshot:d6631f5f2370cfc3a49efce312491031fb387600 - name: RELATED_IMAGE_TEMPO_QUERY value: docker.io/grafana/tempo-query:main-2999520 - name: RELATED_IMAGE_TEMPO_GATEWAY @@ -1574,7 +1574,7 @@ spec: relatedImages: - image: docker.io/grafana/tempo:2.5.0 name: tempo - - image: docker.io/jaegertracing/jaeger-query:1.60 + - image: docker.io/jaegertracing/jaeger-query-snapshot:d6631f5f2370cfc3a49efce312491031fb387600 name: jaeger-query - image: docker.io/grafana/tempo-query:main-2999520 name: tempo-query diff --git a/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml b/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml index 9406e26b4..1d9f1007e 100644 --- a/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml +++ b/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml @@ -74,7 +74,7 @@ metadata: capabilities: Deep Insights categories: Logging & Tracing,Monitoring containerImage: ghcr.io/grafana/tempo-operator/tempo-operator:v0.13.0 - createdAt: "2024-10-03T09:33:23Z" + createdAt: "2024-10-04T12:51:27Z" description: Create and manage deployments of Tempo, a high-scale distributed tracing backend. operatorframework.io/cluster-monitoring: "true" @@ -1435,7 +1435,7 @@ spec: - name: RELATED_IMAGE_TEMPO value: docker.io/grafana/tempo:2.5.0 - name: RELATED_IMAGE_JAEGER_QUERY - value: docker.io/jaegertracing/jaeger-query:1.60 + value: docker.io/jaegertracing/jaeger-query-snapshot:d6631f5f2370cfc3a49efce312491031fb387600 - name: RELATED_IMAGE_TEMPO_QUERY value: docker.io/grafana/tempo-query:main-2999520 - name: RELATED_IMAGE_TEMPO_GATEWAY @@ -1595,7 +1595,7 @@ spec: relatedImages: - image: docker.io/grafana/tempo:2.5.0 name: tempo - - image: docker.io/jaegertracing/jaeger-query:1.60 + - image: docker.io/jaegertracing/jaeger-query-snapshot:d6631f5f2370cfc3a49efce312491031fb387600 name: jaeger-query - image: docker.io/grafana/tempo-query:main-2999520 name: tempo-query diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 2131d8461..ac0263dd1 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -42,7 +42,7 @@ spec: - name: RELATED_IMAGE_TEMPO value: docker.io/grafana/tempo:2.5.0 - name: RELATED_IMAGE_JAEGER_QUERY - value: docker.io/jaegertracing/jaeger-query:1.60 + value: docker.io/jaegertracing/jaeger-query-snapshot:d6631f5f2370cfc3a49efce312491031fb387600 - name: RELATED_IMAGE_TEMPO_QUERY value: docker.io/grafana/tempo-query:main-2999520 - name: RELATED_IMAGE_TEMPO_GATEWAY From c7eb01b6d3e002323c9a034a2f0d8cf98994c2fb Mon Sep 17 00:00:00 2001 From: Pavol Loffay Date: Mon, 7 Oct 2024 09:24:37 +0200 Subject: [PATCH 03/10] Expose a single timeout setting in CRDs (#1045) Signed-off-by: Pavol Loffay --- .chloggen/timeout.yaml | 18 +++ Makefile | 2 +- .../v1alpha1/tempomonolithic_defaults.go | 5 + .../v1alpha1/tempomonolithic_defaults_test.go | 13 ++- apis/tempo/v1alpha1/tempomonolithic_types.go | 5 + apis/tempo/v1alpha1/tempostack_types.go | 5 + apis/tempo/v1alpha1/zz_generated.deepcopy.go | 2 + .../tempo-operator.clusterserviceversion.yaml | 6 +- .../tempo.grafana.com_tempomonolithics.yaml | 6 + .../tempo.grafana.com_tempostacks.yaml | 6 + .../tempo-operator.clusterserviceversion.yaml | 6 +- .../tempo.grafana.com_tempomonolithics.yaml | 6 + .../tempo.grafana.com_tempostacks.yaml | 6 + .../tempo.grafana.com_tempomonolithics.yaml | 6 + .../bases/tempo.grafana.com_tempostacks.yaml | 6 + config/manager/manager.yaml | 2 +- .../tempo.grafana.com_tempomonolithics.yaml | 1 + docs/spec/tempo.grafana.com_tempostacks.yaml | 1 + internal/manifests/config/build.go | 1 + internal/manifests/config/build_test.go | 98 +++++++++------- internal/manifests/config/options.go | 7 +- internal/manifests/config/tempo-config.yaml | 4 +- internal/manifests/gateway/gateway.go | 1 + internal/manifests/gateway/gateway_test.go | 6 +- internal/manifests/gateway/openshift.go | 12 +- internal/manifests/manifests_test.go | 2 + internal/manifests/monolithic/build.go | 1 + internal/manifests/monolithic/configmap.go | 10 +- .../manifests/monolithic/configmap_test.go | 18 ++- .../manifests/monolithic/jaegerui_ingress.go | 12 +- .../monolithic/jaegerui_ingress_test.go | 6 + internal/manifests/monolithic/statefulset.go | 1 + .../manifests/monolithic/statefulset_test.go | 3 + internal/manifests/oauthproxy/oauth_proxy.go | 24 ++-- .../manifests/oauthproxy/oauth_proxy_test.go | 6 + .../manifests/queryfrontend/query_frontend.go | 19 +++- .../queryfrontend/query_frontend_test.go | 6 + internal/webhooks/tempostack_webhook.go | 5 + internal/webhooks/tempostack_webhook_test.go | 107 ++++++++++++++++++ .../install-tempo-assert.yaml | 1 + .../scale-tempo-assert.yaml | 1 + .../install-tempo-assert.yaml | 22 ++++ .../monolithic-route/install-tempo.yaml | 1 + .../install-tempo-assert.yaml | 1 + .../e2e-openshift/multitenancy/01-assert.yaml | 1 + .../route/install-tempo-assert.yaml | 1 + tests/e2e-openshift/route/install-tempo.yaml | 1 + .../01-assert.yaml | 1 + .../install-tempo-assert.yaml | 2 + .../install-tempostack-assert.yaml | 8 +- .../install-tempostack.yaml | 4 + 51 files changed, 417 insertions(+), 78 deletions(-) create mode 100755 .chloggen/timeout.yaml diff --git a/.chloggen/timeout.yaml b/.chloggen/timeout.yaml new file mode 100755 index 000000000..e8de1bf44 --- /dev/null +++ b/.chloggen/timeout.yaml @@ -0,0 +1,18 @@ +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: breaking + +# The name of the component, or a single word describing the area of concern, (e.g. tempostack, tempomonolithic, github action) +component: tempostack, tempomonolithic + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Add unified timeout configuration. It changes the default to 30s. + +# One or more tracking issues related to the change +issues: [1045] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: | + Adding `spec.timeout` CRD option to configure timeout on all components and default it to 30s. + Before Tempo server was defaulting to 3m, gateway to 2m, OpenShift route to 30s (for query), oauth-proxy to 30s (for query). diff --git a/Makefile b/Makefile index fe152e9c0..f0a5bd809 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ TEMPO_QUERY_VERSION ?= main-2999520 JAEGER_QUERY_VERSION ?= d6631f5f2370cfc3a49efce312491031fb387600 TEMPO_GATEWAY_VERSION ?= main-2024-08-05-11d0d94 TEMPO_GATEWAY_OPA_VERSION ?= main-2024-04-29-914c13f -OAUTH_PROXY_VERSION=4.12 +OAUTH_PROXY_VERSION=4.14 MIN_KUBERNETES_VERSION ?= 1.25.0 MIN_OPENSHIFT_VERSION ?= 4.12 diff --git a/apis/tempo/v1alpha1/tempomonolithic_defaults.go b/apis/tempo/v1alpha1/tempomonolithic_defaults.go index 7dfbd738e..39fc395f9 100644 --- a/apis/tempo/v1alpha1/tempomonolithic_defaults.go +++ b/apis/tempo/v1alpha1/tempomonolithic_defaults.go @@ -16,6 +16,7 @@ var ( twoGBQuantity = resource.MustParse("2Gi") tenGBQuantity = resource.MustParse("10Gi") defaultServicesDuration = metav1.Duration{Duration: time.Hour * 24 * 3} + defaultTimeout = metav1.Duration{Duration: time.Second * 30} ) // Default sets all default values in a central place, instead of setting it at every place where the value is accessed. @@ -88,4 +89,8 @@ func (r *TempoMonolithic) Default(ctrlConfig configv1alpha1.ProjectConfig) { r.Spec.JaegerUI.ServicesQueryDuration = &defaultServicesDuration } } + + if r.Spec.Timeout.Duration == 0 { + r.Spec.Timeout = defaultTimeout + } } diff --git a/apis/tempo/v1alpha1/tempomonolithic_defaults_test.go b/apis/tempo/v1alpha1/tempomonolithic_defaults_test.go index c41828ef6..be28ff060 100644 --- a/apis/tempo/v1alpha1/tempomonolithic_defaults_test.go +++ b/apis/tempo/v1alpha1/tempomonolithic_defaults_test.go @@ -6,6 +6,7 @@ import ( "github.com/stretchr/testify/assert" "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" configv1alpha1 "github.com/grafana/tempo-operator/apis/config/v1alpha1" @@ -46,6 +47,7 @@ func TestMonolithicDefault(t *testing.T) { }, }, Management: "Managed", + Timeout: metav1.Duration{Duration: time.Second * 30}, }, }, }, @@ -79,6 +81,7 @@ func TestMonolithicDefault(t *testing.T) { }, }, Management: "Managed", + Timeout: metav1.Duration{Duration: time.Second * 30}, }, }, }, @@ -104,6 +107,7 @@ func TestMonolithicDefault(t *testing.T) { }, }, Management: "Unmanaged", + Timeout: metav1.Duration{Duration: time.Second * 30}, }, }, expected: &TempoMonolithic{ @@ -125,6 +129,7 @@ func TestMonolithicDefault(t *testing.T) { }, }, Management: "Unmanaged", + Timeout: metav1.Duration{Duration: time.Second * 30}, }, }, }, @@ -194,6 +199,7 @@ func TestMonolithicDefault(t *testing.T) { ServicesQueryDuration: &defaultServicesDuration, }, Management: "Managed", + Timeout: metav1.Duration{Duration: time.Second * 30}, }, }, }, @@ -266,6 +272,7 @@ func TestMonolithicDefault(t *testing.T) { ServicesQueryDuration: &defaultServicesDuration, }, Management: "Managed", + Timeout: metav1.Duration{Duration: time.Second * 30}, }, }, }, @@ -330,6 +337,7 @@ func TestMonolithicDefault(t *testing.T) { ServicesQueryDuration: &defaultServicesDuration, }, Management: "Managed", + Timeout: metav1.Duration{Duration: time.Second * 30}, }, }, }, @@ -393,11 +401,12 @@ func TestMonolithicDefault(t *testing.T) { ServicesQueryDuration: &defaultServicesDuration, }, Management: "Managed", + Timeout: metav1.Duration{Duration: time.Second * 30}, }, }, }, { - name: "define custom duration for services list", + name: "define custom duration for services list and timeout", input: &TempoMonolithic{ ObjectMeta: v1.ObjectMeta{ Name: "test", @@ -417,6 +426,7 @@ func TestMonolithicDefault(t *testing.T) { }, ServicesQueryDuration: &v1.Duration{Duration: time.Duration(100 * 100)}, }, + Timeout: metav1.Duration{Duration: time.Hour}, }, }, expected: &TempoMonolithic{ @@ -454,6 +464,7 @@ func TestMonolithicDefault(t *testing.T) { ServicesQueryDuration: &v1.Duration{Duration: time.Duration(100 * 100)}, }, Management: "Managed", + Timeout: metav1.Duration{Duration: time.Hour}, }, }, }, diff --git a/apis/tempo/v1alpha1/tempomonolithic_types.go b/apis/tempo/v1alpha1/tempomonolithic_types.go index 1d56c4400..fef946876 100644 --- a/apis/tempo/v1alpha1/tempomonolithic_types.go +++ b/apis/tempo/v1alpha1/tempomonolithic_types.go @@ -44,6 +44,11 @@ type TempoMonolithicSpec struct { // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Resources",order=5,xDescriptors="urn:alm:descriptor:com.tectonic.ui:resourceRequirements" Resources *corev1.ResourceRequirements `json:"resources,omitempty"` + // Timeout configures the same timeout on all components starting at ingress down to the ingestor/querier. + // Timeout configuration on a specific component has a higher precedence. + // Default is 30 seconds. + Timeout metav1.Duration `json:"timeout,omitempty"` + // ServiceAccount defines the Service Account to use for all Tempo components. // // +kubebuilder:validation:Optional diff --git a/apis/tempo/v1alpha1/tempostack_types.go b/apis/tempo/v1alpha1/tempostack_types.go index 447b93fee..cc0186ab0 100644 --- a/apis/tempo/v1alpha1/tempostack_types.go +++ b/apis/tempo/v1alpha1/tempostack_types.go @@ -40,6 +40,11 @@ type TempoStackSpec struct { // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Ingestion and Querying Ratelimiting" LimitSpec LimitSpec `json:"limits,omitempty"` + // Timeout configures the same timeout on all components starting at ingress down to the ingestor/querier. + // Timeout configuration on a specific component has a higher precedence. + // Defaults to 30 seconds. + Timeout metav1.Duration `json:"timeout,omitempty"` + // StorageClassName for PVCs used by ingester. Defaults to nil (default storage class in the cluster). // // +optional diff --git a/apis/tempo/v1alpha1/zz_generated.deepcopy.go b/apis/tempo/v1alpha1/zz_generated.deepcopy.go index d9bbe161f..f5a9d2534 100644 --- a/apis/tempo/v1alpha1/zz_generated.deepcopy.go +++ b/apis/tempo/v1alpha1/zz_generated.deepcopy.go @@ -1367,6 +1367,7 @@ func (in *TempoMonolithicSpec) DeepCopyInto(out *TempoMonolithicSpec) { *out = new(v1.ResourceRequirements) (*in).DeepCopyInto(*out) } + out.Timeout = in.Timeout if in.ExtraConfig != nil { in, out := &in.ExtraConfig, &out.ExtraConfig *out = new(ExtraConfigSpec) @@ -1508,6 +1509,7 @@ func (in *TempoStackList) DeepCopyObject() runtime.Object { func (in *TempoStackSpec) DeepCopyInto(out *TempoStackSpec) { *out = *in in.LimitSpec.DeepCopyInto(&out.LimitSpec) + out.Timeout = in.Timeout if in.StorageClassName != nil { in, out := &in.StorageClassName, &out.StorageClassName *out = new(string) diff --git a/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml b/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml index b623f6b16..85bb37bf8 100644 --- a/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml +++ b/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml @@ -74,7 +74,7 @@ metadata: capabilities: Deep Insights categories: Logging & Tracing,Monitoring containerImage: ghcr.io/grafana/tempo-operator/tempo-operator:v0.13.0 - createdAt: "2024-10-04T12:51:28Z" + createdAt: "2024-10-07T07:11:28Z" description: Create and manage deployments of Tempo, a high-scale distributed tracing backend. operatorframework.io/cluster-monitoring: "true" @@ -1433,7 +1433,7 @@ spec: - name: RELATED_IMAGE_TEMPO_GATEWAY_OPA value: quay.io/observatorium/opa-openshift:main-2024-04-29-914c13f - name: RELATED_IMAGE_OAUTH_PROXY - value: quay.io/openshift/origin-oauth-proxy:4.12 + value: quay.io/openshift/origin-oauth-proxy:4.14 image: ghcr.io/grafana/tempo-operator/tempo-operator:v0.13.0 livenessProbe: httpGet: @@ -1582,7 +1582,7 @@ spec: name: tempo-gateway - image: quay.io/observatorium/opa-openshift:main-2024-04-29-914c13f name: tempo-gateway-opa - - image: quay.io/openshift/origin-oauth-proxy:4.12 + - image: quay.io/openshift/origin-oauth-proxy:4.14 name: oauth-proxy version: 0.13.0 webhookdefinitions: diff --git a/bundle/community/manifests/tempo.grafana.com_tempomonolithics.yaml b/bundle/community/manifests/tempo.grafana.com_tempomonolithics.yaml index ae585d652..115daff46 100644 --- a/bundle/community/manifests/tempo.grafana.com_tempomonolithics.yaml +++ b/bundle/community/manifests/tempo.grafana.com_tempomonolithics.yaml @@ -1683,6 +1683,12 @@ spec: required: - traces type: object + timeout: + description: |- + Timeout configures the same timeout on all components starting at ingress down to the ingestor/querier. + Timeout configuration on a specific component has a higher precedence. + Default is 30 seconds. + type: string tolerations: description: Tolerations defines the tolerations of a node to schedule the pod onto it. diff --git a/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml b/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml index 27f123dca..eaede14e7 100644 --- a/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml +++ b/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml @@ -2715,6 +2715,12 @@ spec: required: - mode type: object + timeout: + description: |- + Timeout configures the same timeout on all components starting at ingress down to the ingestor/querier. + Timeout configuration on a specific component has a higher precedence. + Defaults to 30 seconds. + type: string required: - storage type: object diff --git a/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml b/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml index 1d9f1007e..a4868857e 100644 --- a/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml +++ b/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml @@ -74,7 +74,7 @@ metadata: capabilities: Deep Insights categories: Logging & Tracing,Monitoring containerImage: ghcr.io/grafana/tempo-operator/tempo-operator:v0.13.0 - createdAt: "2024-10-04T12:51:27Z" + createdAt: "2024-10-07T07:11:27Z" description: Create and manage deployments of Tempo, a high-scale distributed tracing backend. operatorframework.io/cluster-monitoring: "true" @@ -1443,7 +1443,7 @@ spec: - name: RELATED_IMAGE_TEMPO_GATEWAY_OPA value: quay.io/observatorium/opa-openshift:main-2024-04-29-914c13f - name: RELATED_IMAGE_OAUTH_PROXY - value: quay.io/openshift/origin-oauth-proxy:4.12 + value: quay.io/openshift/origin-oauth-proxy:4.14 image: ghcr.io/grafana/tempo-operator/tempo-operator:v0.13.0 livenessProbe: httpGet: @@ -1603,7 +1603,7 @@ spec: name: tempo-gateway - image: quay.io/observatorium/opa-openshift:main-2024-04-29-914c13f name: tempo-gateway-opa - - image: quay.io/openshift/origin-oauth-proxy:4.12 + - image: quay.io/openshift/origin-oauth-proxy:4.14 name: oauth-proxy version: 0.13.0 webhookdefinitions: diff --git a/bundle/openshift/manifests/tempo.grafana.com_tempomonolithics.yaml b/bundle/openshift/manifests/tempo.grafana.com_tempomonolithics.yaml index ae585d652..115daff46 100644 --- a/bundle/openshift/manifests/tempo.grafana.com_tempomonolithics.yaml +++ b/bundle/openshift/manifests/tempo.grafana.com_tempomonolithics.yaml @@ -1683,6 +1683,12 @@ spec: required: - traces type: object + timeout: + description: |- + Timeout configures the same timeout on all components starting at ingress down to the ingestor/querier. + Timeout configuration on a specific component has a higher precedence. + Default is 30 seconds. + type: string tolerations: description: Tolerations defines the tolerations of a node to schedule the pod onto it. diff --git a/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml b/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml index 27f123dca..eaede14e7 100644 --- a/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml +++ b/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml @@ -2715,6 +2715,12 @@ spec: required: - mode type: object + timeout: + description: |- + Timeout configures the same timeout on all components starting at ingress down to the ingestor/querier. + Timeout configuration on a specific component has a higher precedence. + Defaults to 30 seconds. + type: string required: - storage type: object diff --git a/config/crd/bases/tempo.grafana.com_tempomonolithics.yaml b/config/crd/bases/tempo.grafana.com_tempomonolithics.yaml index 4d28c00ea..92a7a6753 100644 --- a/config/crd/bases/tempo.grafana.com_tempomonolithics.yaml +++ b/config/crd/bases/tempo.grafana.com_tempomonolithics.yaml @@ -1679,6 +1679,12 @@ spec: required: - traces type: object + timeout: + description: |- + Timeout configures the same timeout on all components starting at ingress down to the ingestor/querier. + Timeout configuration on a specific component has a higher precedence. + Default is 30 seconds. + type: string tolerations: description: Tolerations defines the tolerations of a node to schedule the pod onto it. diff --git a/config/crd/bases/tempo.grafana.com_tempostacks.yaml b/config/crd/bases/tempo.grafana.com_tempostacks.yaml index 2cdfc2c3b..f1374fbd5 100644 --- a/config/crd/bases/tempo.grafana.com_tempostacks.yaml +++ b/config/crd/bases/tempo.grafana.com_tempostacks.yaml @@ -2711,6 +2711,12 @@ spec: required: - mode type: object + timeout: + description: |- + Timeout configures the same timeout on all components starting at ingress down to the ingestor/querier. + Timeout configuration on a specific component has a higher precedence. + Defaults to 30 seconds. + type: string required: - storage type: object diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index ac0263dd1..accf9ad74 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -50,7 +50,7 @@ spec: - name: RELATED_IMAGE_TEMPO_GATEWAY_OPA value: quay.io/observatorium/opa-openshift:main-2024-04-29-914c13f - name: RELATED_IMAGE_OAUTH_PROXY - value: quay.io/openshift/origin-oauth-proxy:4.12 + value: quay.io/openshift/origin-oauth-proxy:4.14 securityContext: allowPrivilegeEscalation: false capabilities: diff --git a/docs/spec/tempo.grafana.com_tempomonolithics.yaml b/docs/spec/tempo.grafana.com_tempomonolithics.yaml index 3ab1b69a4..487a6cfd7 100644 --- a/docs/spec/tempo.grafana.com_tempomonolithics.yaml +++ b/docs/spec/tempo.grafana.com_tempomonolithics.yaml @@ -126,6 +126,7 @@ spec: # TempoMonolithicSpec defines the desir certName: "" # Cert is the name of a Secret containing a certificate (tls.crt) and private key (tls.key). It needs to be in the same namespace as the Tempo custom resource. minVersion: "" # MinVersion defines the minimum acceptable TLS version. size: 0Gi # Size defines the size of the volume where traces are stored. For in-memory storage, this defines the size of the tmpfs volume. For persistent volume storage, this defines the size of the persistent volume. For object storage, this defines the size of the persistent volume containing the Write-Ahead Log (WAL) of Tempo. Default: 2Gi for memory, 10Gi for all other backends. + timeout: "" # Timeout configures the same timeout on all components starting at ingress down to the ingestor/querier. Timeout configuration on a specific component has a higher precedence. Default is 30 seconds. affinity: # Affinity defines the Affinity rules for scheduling pods. nodeAffinity: {} # Describes node affinity scheduling rules for the pod. podAffinity: {} # Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). diff --git a/docs/spec/tempo.grafana.com_tempostacks.yaml b/docs/spec/tempo.grafana.com_tempostacks.yaml index 4ac207289..2b65cc6a7 100644 --- a/docs/spec/tempo.grafana.com_tempostacks.yaml +++ b/docs/spec/tempo.grafana.com_tempostacks.yaml @@ -388,6 +388,7 @@ spec: # TempoStackSpec defines the desired st resources: - "" mode: "static" # Mode defines the multitenancy mode. + timeout: "" # Timeout configures the same timeout on all components starting at ingress down to the ingestor/querier. Timeout configuration on a specific component has a higher precedence. Defaults to 30 seconds. resources: # Resources defines resources configuration. total: # The total amount of resources for Tempo instance. The operator autonomously splits resources between deployed Tempo components. Only limits are supported, the operator calculates requests automatically. See http://github.com/grafana/tempo/issues/1540. claims: # Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. This is an alpha field and requires enabling the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. diff --git a/internal/manifests/config/build.go b/internal/manifests/config/build.go index 489a2c9fe..a20a60d78 100644 --- a/internal/manifests/config/build.go +++ b/internal/manifests/config/build.go @@ -89,6 +89,7 @@ func buildConfiguration(params manifestutils.Params) ([]byte, error) { TLS: tlsopts, ReceiverTLS: buildReceiverTLSConfig(tempo), S3StorageTLS: buildS3StorageTLSConfig(params), + Timeout: params.Tempo.Spec.Timeout.Duration, } if isTenantOverridesConfigRequired(tempo.Spec.LimitSpec) { diff --git a/internal/manifests/config/build_test.go b/internal/manifests/config/build_test.go index 62c50571e..23623d48d 100644 --- a/internal/manifests/config/build_test.go +++ b/internal/manifests/config/build_test.go @@ -74,8 +74,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 30s + http_server_write_timeout: 30s log_format: logfmt storage: trace: @@ -103,6 +103,7 @@ query_frontend: Name: "test", }, Spec: v1alpha1.TempoStackSpec{ + Timeout: metav1.Duration{Duration: time.Second * 30}, Storage: v1alpha1.ObjectStorageSpec{ Secret: v1alpha1.ObjectStorageSecretSpec{ Type: v1alpha1.ObjectStorageSecretS3, @@ -197,8 +198,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -286,8 +287,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -374,8 +375,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -463,8 +464,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -552,8 +553,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -641,8 +642,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -730,8 +731,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -831,8 +832,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -930,8 +931,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -964,6 +965,7 @@ query_frontend: Name: "test", }, Spec: v1alpha1.TempoStackSpec{ + Timeout: metav1.Duration{Duration: time.Minute * 3}, Storage: v1alpha1.ObjectStorageSpec{ Secret: v1alpha1.ObjectStorageSecretSpec{ Type: v1alpha1.ObjectStorageSecretS3, @@ -1091,8 +1093,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -1124,6 +1126,7 @@ query_frontend: Name: "test", }, Spec: v1alpha1.TempoStackSpec{ + Timeout: metav1.Duration{Duration: time.Minute * 3}, Storage: v1alpha1.ObjectStorageSpec{ Secret: v1alpha1.ObjectStorageSecretSpec{ Type: v1alpha1.ObjectStorageSecretGCS, @@ -1205,8 +1208,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -1233,6 +1236,7 @@ query_frontend: Name: "test", }, Spec: v1alpha1.TempoStackSpec{ + Timeout: metav1.Duration{Duration: time.Minute * 3}, Storage: v1alpha1.ObjectStorageSpec{ Secret: v1alpha1.ObjectStorageSecretSpec{ Type: v1alpha1.ObjectStorageSecretAzure, @@ -1309,8 +1313,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -1338,6 +1342,7 @@ query_frontend: Name: "test", }, Spec: v1alpha1.TempoStackSpec{ + Timeout: metav1.Duration{Duration: time.Minute * 3}, Storage: v1alpha1.ObjectStorageSpec{ Secret: v1alpha1.ObjectStorageSecretSpec{ Type: v1alpha1.ObjectStorageSecretS3, @@ -1455,8 +1460,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt tls_cipher_suites: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 tls_min_version: VersionTLS12 @@ -1574,8 +1579,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt tls_min_version: VersionTLS13 grpc_tls_config: @@ -1630,6 +1635,7 @@ ingester_client: Namespace: "nstest", }, Spec: v1alpha1.TempoStackSpec{ + Timeout: metav1.Duration{Duration: time.Minute * 3}, Storage: v1alpha1.ObjectStorageSpec{ Secret: v1alpha1.ObjectStorageSecretSpec{ Type: v1alpha1.ObjectStorageSecretS3, @@ -1737,8 +1743,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt tls_cipher_suites: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 tls_min_version: VersionTLS12 @@ -1789,6 +1795,7 @@ ingester_client: Namespace: "nstest", }, Spec: v1alpha1.TempoStackSpec{ + Timeout: metav1.Duration{Duration: 3 * time.Minute}, Template: v1alpha1.TempoTemplateSpec{ Gateway: v1alpha1.TempoGatewaySpec{ Enabled: true, @@ -1902,8 +1909,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -2010,8 +2017,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -2124,8 +2131,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -2158,6 +2165,7 @@ query_frontend: Name: "test", }, Spec: v1alpha1.TempoStackSpec{ + Timeout: metav1.Duration{Duration: time.Minute * 3}, Storage: v1alpha1.ObjectStorageSpec{ Secret: v1alpha1.ObjectStorageSecretSpec{ Type: v1alpha1.ObjectStorageSecretS3, @@ -2257,8 +2265,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -2340,8 +2348,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -2374,6 +2382,7 @@ query_frontend: Name: "test", }, Spec: v1alpha1.TempoStackSpec{ + Timeout: metav1.Duration{Duration: 3 * time.Minute}, Storage: v1alpha1.ObjectStorageSpec{ Secret: v1alpha1.ObjectStorageSecretSpec{ Type: v1alpha1.ObjectStorageSecretS3, @@ -2464,8 +2473,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -2492,6 +2501,7 @@ query_frontend: Name: "test", }, Spec: v1alpha1.TempoStackSpec{ + Timeout: metav1.Duration{Duration: time.Minute * 3}, Storage: v1alpha1.ObjectStorageSpec{ Secret: v1alpha1.ObjectStorageSecretSpec{ Type: v1alpha1.ObjectStorageSecretS3, diff --git a/internal/manifests/config/options.go b/internal/manifests/config/options.go index c3a220b3b..02ce67481 100644 --- a/internal/manifests/config/options.go +++ b/internal/manifests/config/options.go @@ -1,6 +1,10 @@ package config -import "github.com/grafana/tempo-operator/internal/manifests/manifestutils" +import ( + "time" + + "github.com/grafana/tempo-operator/internal/manifests/manifestutils" +) // options holds the configuration template options. type options struct { @@ -19,6 +23,7 @@ type options struct { Gates featureGates ReceiverTLS receiverTLSOptions S3StorageTLS storageTLSOptions + Timeout time.Duration } type tempoQueryOptions struct { diff --git a/internal/manifests/config/tempo-config.yaml b/internal/manifests/config/tempo-config.yaml index f43141207..cee5abb49 100644 --- a/internal/manifests/config/tempo-config.yaml +++ b/internal/manifests/config/tempo-config.yaml @@ -182,8 +182,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: {{ .Timeout }} + http_server_write_timeout: {{ .Timeout }} log_format: logfmt {{- if or .Gates.GRPCEncryption .Gates.HTTPEncryption }} {{- if .TLS.Profile.Ciphers }} diff --git a/internal/manifests/gateway/gateway.go b/internal/manifests/gateway/gateway.go index 5b778475d..aff673e8a 100644 --- a/internal/manifests/gateway/gateway.go +++ b/internal/manifests/gateway/gateway.go @@ -248,6 +248,7 @@ func deployment(params manifestutils.Params, rbacCfgHash string, tenantsCfgHash fmt.Sprintf("--web.internal.listen=0.0.0.0:%d", manifestutils.GatewayPortInternalHTTPServer), // serves health checks fmt.Sprintf("--traces.write.otlpgrpc.endpoint=%s:%d", naming.ServiceFqdn(tempo.Namespace, tempo.Name, manifestutils.DistributorComponentName), manifestutils.PortOtlpGrpcServer), // Tempo Distributor gRPC upstream fmt.Sprintf("--traces.write.otlphttp.endpoint=%s://%s:%d", httpScheme(params.CtrlConfig.Gates.HTTPEncryption), naming.ServiceFqdn(tempo.Namespace, tempo.Name, manifestutils.DistributorComponentName), manifestutils.PortOtlpHttp), // Tempo Distributor HTTP upstream + fmt.Sprintf("--traces.write-timeout=%s", params.Tempo.Spec.Timeout.Duration.String()), fmt.Sprintf("--traces.tempo.endpoint=%s://%s:%d", httpScheme(params.CtrlConfig.Gates.HTTPEncryption), naming.ServiceFqdn(tempo.Namespace, tempo.Name, manifestutils.QueryFrontendComponentName), manifestutils.PortHTTPServer), // Tempo API upstream fmt.Sprintf("--grpc.listen=0.0.0.0:%d", manifestutils.GatewayPortGRPCServer), // proxies Tempo Distributor gRPC diff --git a/internal/manifests/gateway/gateway_test.go b/internal/manifests/gateway/gateway_test.go index 5439d07a7..3feca416b 100644 --- a/internal/manifests/gateway/gateway_test.go +++ b/internal/manifests/gateway/gateway_test.go @@ -2,7 +2,6 @@ package gateway import ( "fmt" - "net" "reflect" "testing" @@ -249,7 +248,7 @@ func TestBuildGateway_openshift(t *testing.T) { require.True(t, ok) require.Equal(t, "Service", route.Spec.To.Kind) require.Equal(t, "tempo-simplest-gateway", route.Spec.To.Name) - require.Equal(t, map[string]string{"timeout": "30s"}, route.ObjectMeta.Annotations) + require.Equal(t, map[string]string{"timeout": "30s", "haproxy.router.openshift.io/timeout": "0s"}, route.ObjectMeta.Annotations) obj = getObjectByTypeAndName(objects, "tempo-simplest-gateway-cabundle", reflect.TypeOf(&corev1.ConfigMap{})) require.NotNil(t, obj) @@ -775,6 +774,9 @@ func TestRoute(t *testing.T) { Name: naming.Name(manifestutils.GatewayComponentName, "test"), Namespace: "project1", Labels: manifestutils.ComponentLabels("gateway", "test"), + Annotations: map[string]string{ + "haproxy.router.openshift.io/timeout": "0s", + }, }, Spec: routev1.RouteSpec{ To: routev1.RouteTargetReference{ diff --git a/internal/manifests/gateway/openshift.go b/internal/manifests/gateway/openshift.go index 858564042..7878db200 100644 --- a/internal/manifests/gateway/openshift.go +++ b/internal/manifests/gateway/openshift.go @@ -22,6 +22,8 @@ import ( const ( gatewayOPAHTTPPort = 8082 gatewayOPAInternalPort = 8083 + + timeoutRouteAnnotation = "haproxy.router.openshift.io/timeout" ) // BuildServiceAccountAnnotations returns the annotations to use a ServiceAccount as an OAuth client. @@ -111,12 +113,20 @@ func route(tempo v1alpha1.TempoStack) (*routev1.Route, error) { return nil, fmt.Errorf("unsupported tls termination specified for route") } + annotations := tempo.Spec.Template.Gateway.Ingress.Annotations + if annotations == nil { + annotations = map[string]string{} + } + if annotations[timeoutRouteAnnotation] == "" { + annotations[timeoutRouteAnnotation] = fmt.Sprintf("%ds", int(tempo.Spec.Timeout.Duration.Seconds())) + } + return &routev1.Route{ ObjectMeta: metav1.ObjectMeta{ Name: naming.Name(manifestutils.GatewayComponentName, tempo.Name), Namespace: tempo.Namespace, Labels: labels, - Annotations: tempo.Spec.Template.Gateway.Ingress.Annotations, + Annotations: annotations, }, Spec: routev1.RouteSpec{ Host: tempo.Spec.Template.Gateway.Ingress.Host, diff --git a/internal/manifests/manifests_test.go b/internal/manifests/manifests_test.go index a76404b6f..dfeeddeff 100644 --- a/internal/manifests/manifests_test.go +++ b/internal/manifests/manifests_test.go @@ -2,6 +2,7 @@ package manifests import ( "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -33,6 +34,7 @@ func TestBuildAll(t *testing.T) { Namespace: "project1", }, Spec: v1alpha1.TempoStackSpec{ + Timeout: metav1.Duration{Duration: time.Second * 5}, Template: v1alpha1.TempoTemplateSpec{ Gateway: v1alpha1.TempoGatewaySpec{ Enabled: true, diff --git a/internal/manifests/monolithic/build.go b/internal/manifests/monolithic/build.go index b084eeed4..3c16670ce 100644 --- a/internal/manifests/monolithic/build.go +++ b/internal/manifests/monolithic/build.go @@ -96,6 +96,7 @@ func BuildAll(opts Options) ([]client.Object, error) { oauthproxy.PatchStatefulSetForOauthProxy( tempo.ObjectMeta, tempo.Spec.JaegerUI.Authentication, + tempo.Spec.Timeout.Duration, opts.CtrlConfig, statefulSet) oauthproxy.PatchQueryFrontEndService(getJaegerUIService(services, tempo), tempo.Name) diff --git a/internal/manifests/monolithic/configmap.go b/internal/manifests/monolithic/configmap.go index 967414f61..8c1a1ea94 100644 --- a/internal/manifests/monolithic/configmap.go +++ b/internal/manifests/monolithic/configmap.go @@ -54,9 +54,11 @@ type tempoConfig struct { MultitenancyEnabled bool `yaml:"multitenancy_enabled,omitempty"` Server struct { - HTTPListenAddress string `yaml:"http_listen_address,omitempty"` - HttpListenPort int `yaml:"http_listen_port,omitempty"` - GRPCListenAddress string `yaml:"grpc_listen_address,omitempty"` + HTTPListenAddress string `yaml:"http_listen_address,omitempty"` + HttpListenPort int `yaml:"http_listen_port,omitempty"` + GRPCListenAddress string `yaml:"grpc_listen_address,omitempty"` + HttpServerReadTimeout time.Duration `yaml:"http_server_read_timeout,omitempty"` + HttpServerWriteTimeout time.Duration `yaml:"http_server_write_timeout,omitempty"` } `yaml:"server"` InternalServer struct { @@ -170,6 +172,8 @@ func buildTempoConfig(opts Options) ([]byte, error) { config := tempoConfig{} config.MultitenancyEnabled = tempo.Spec.Multitenancy != nil && tempo.Spec.Multitenancy.Enabled config.Server.HttpListenPort = manifestutils.PortHTTPServer + config.Server.HttpServerReadTimeout = opts.Tempo.Spec.Timeout.Duration + config.Server.HttpServerWriteTimeout = opts.Tempo.Spec.Timeout.Duration if tempo.Spec.Multitenancy.IsGatewayEnabled() { // all connections to tempo must go via gateway config.Server.HTTPListenAddress = "localhost" diff --git a/internal/manifests/monolithic/configmap_test.go b/internal/manifests/monolithic/configmap_test.go index f7f31270f..7eab5b3f3 100644 --- a/internal/manifests/monolithic/configmap_test.go +++ b/internal/manifests/monolithic/configmap_test.go @@ -80,6 +80,8 @@ func TestBuildConfig(t *testing.T) { expected: ` server: http_listen_port: 3200 + http_server_read_timeout: 30s + http_server_write_timeout: 30s internal_server: enable: true http_listen_address: 0.0.0.0 @@ -112,6 +114,8 @@ usage_report: expected: ` server: http_listen_port: 3200 + http_server_read_timeout: 30s + http_server_write_timeout: 30s internal_server: enable: true http_listen_address: 0.0.0.0 @@ -155,6 +159,8 @@ usage_report: expected: ` server: http_listen_port: 3200 + http_server_read_timeout: 30s + http_server_write_timeout: 30s internal_server: enable: true http_listen_address: 0.0.0.0 @@ -202,6 +208,8 @@ usage_report: expected: ` server: http_listen_port: 3200 + http_server_read_timeout: 30s + http_server_write_timeout: 30s internal_server: enable: true http_listen_address: 0.0.0.0 @@ -255,6 +263,8 @@ usage_report: expected: ` server: http_listen_port: 3200 + http_server_read_timeout: 30s + http_server_write_timeout: 30s internal_server: enable: true http_listen_address: 0.0.0.0 @@ -313,6 +323,8 @@ usage_report: expected: ` server: http_listen_port: 3200 + http_server_read_timeout: 30s + http_server_write_timeout: 30s internal_server: enable: true http_listen_address: 0.0.0.0 @@ -369,6 +381,8 @@ usage_report: expected: ` server: http_listen_port: 3200 + http_server_read_timeout: 30s + http_server_write_timeout: 30s internal_server: enable: true http_listen_address: 0.0.0.0 @@ -397,12 +411,14 @@ usage_report: name: "extra config", spec: v1alpha1.TempoMonolithicSpec{ ExtraConfig: &v1alpha1.ExtraConfigSpec{ - Tempo: apiextensionsv1.JSON{Raw: []byte(`{"storage": {"trace": {"wal": {"overlay_setting": "abc"}}}}`)}, + Tempo: apiextensionsv1.JSON{Raw: []byte(`{"storage": {"trace": {"wal": {"overlay_setting": "abc"}}}, "server": {"http_server_read_timeout": "1m", "http_server_write_timeout": "1m"}}`)}, }, }, expected: ` server: http_listen_port: 3200 + http_server_read_timeout: 1m + http_server_write_timeout: 1m internal_server: enable: true http_listen_address: 0.0.0.0 diff --git a/internal/manifests/monolithic/jaegerui_ingress.go b/internal/manifests/monolithic/jaegerui_ingress.go index 67d1c79a1..2ccb66404 100644 --- a/internal/manifests/monolithic/jaegerui_ingress.go +++ b/internal/manifests/monolithic/jaegerui_ingress.go @@ -69,6 +69,8 @@ func BuildJaegerUIIngress(opts Options) *networkingv1.Ingress { return ingress } +const timeoutRouteAnnotation = "haproxy.router.openshift.io/timeout" + // BuildJaegerUIRoute creates a Route object for Jaeger UI. func BuildJaegerUIRoute(opts Options) (*routev1.Route, error) { tempo := opts.Tempo @@ -89,6 +91,14 @@ func BuildJaegerUIRoute(opts Options) (*routev1.Route, error) { return nil, fmt.Errorf("unsupported tls termination '%s' specified for route", tempo.Spec.JaegerUI.Route.Termination) } + annotations := opts.Tempo.Spec.JaegerUI.Route.Annotations + if annotations == nil { + annotations = map[string]string{} + } + if annotations[timeoutRouteAnnotation] == "" { + annotations[timeoutRouteAnnotation] = fmt.Sprintf("%ds", int(tempo.Spec.Timeout.Duration.Seconds())) + } + return &routev1.Route{ TypeMeta: metav1.TypeMeta{ APIVersion: networkingv1.SchemeGroupVersion.String(), @@ -98,7 +108,7 @@ func BuildJaegerUIRoute(opts Options) (*routev1.Route, error) { Name: naming.Name(manifestutils.JaegerUIComponentName, tempo.Name), Namespace: tempo.Namespace, Labels: labels, - Annotations: tempo.Spec.JaegerUI.Route.Annotations, + Annotations: annotations, }, Spec: routev1.RouteSpec{ Host: tempo.Spec.JaegerUI.Route.Host, diff --git a/internal/manifests/monolithic/jaegerui_ingress_test.go b/internal/manifests/monolithic/jaegerui_ingress_test.go index 96824a432..e5cac15ad 100644 --- a/internal/manifests/monolithic/jaegerui_ingress_test.go +++ b/internal/manifests/monolithic/jaegerui_ingress_test.go @@ -163,6 +163,9 @@ func TestBuildJaegerUIRoute(t *testing.T) { Name: "tempo-sample-jaegerui", Namespace: "default", Labels: labels, + Annotations: map[string]string{ + "haproxy.router.openshift.io/timeout": "30s", + }, }, Spec: routev1.RouteSpec{ Host: "", @@ -221,6 +224,9 @@ func TestBuildJaegerUIRoute(t *testing.T) { Name: "tempo-sample-jaegerui", Namespace: "default", Labels: labels, + Annotations: map[string]string{ + "haproxy.router.openshift.io/timeout": "30s", + }, }, Spec: routev1.RouteSpec{ Host: "", diff --git a/internal/manifests/monolithic/statefulset.go b/internal/manifests/monolithic/statefulset.go index 0ba9ae8b6..b62a466b1 100644 --- a/internal/manifests/monolithic/statefulset.go +++ b/internal/manifests/monolithic/statefulset.go @@ -398,6 +398,7 @@ func configureGateway(opts Options, sts *appsv1.StatefulSet) error { fmt.Sprintf("--web.internal.listen=0.0.0.0:%d", manifestutils.GatewayPortInternalHTTPServer), // serves health checks fmt.Sprintf("--traces.tenant-header=%s", manifestutils.TenantHeader), fmt.Sprintf("--traces.tempo.endpoint=http://localhost:%d", manifestutils.PortHTTPServer), // Tempo API upstream + fmt.Sprintf("--traces.write-timeout=%s", opts.Tempo.Spec.Timeout.Duration.String()), fmt.Sprintf("--rbac.config=%s", path.Join(gatewayMountDir, "rbac", manifestutils.GatewayRBACFileName)), fmt.Sprintf("--tenants.config=%s", path.Join(gatewayMountDir, "tenants", manifestutils.GatewayTenantFileName)), "--log.level=info", diff --git a/internal/manifests/monolithic/statefulset_test.go b/internal/manifests/monolithic/statefulset_test.go index b2651704e..aa0239afc 100644 --- a/internal/manifests/monolithic/statefulset_test.go +++ b/internal/manifests/monolithic/statefulset_test.go @@ -2,6 +2,7 @@ package monolithic import ( "testing" + "time" "github.com/operator-framework/operator-lib/proxy" "github.com/stretchr/testify/require" @@ -732,6 +733,7 @@ func TestStatefulsetGateway(t *testing.T) { Namespace: "default", }, Spec: v1alpha1.TempoMonolithicSpec{ + Timeout: metav1.Duration{Duration: time.Second * 5}, Storage: &v1alpha1.MonolithicStorageSpec{ Traces: v1alpha1.MonolithicTracesStorageSpec{ Backend: "memory", @@ -788,6 +790,7 @@ func TestStatefulsetGateway(t *testing.T) { "--web.internal.listen=0.0.0.0:8081", "--traces.tenant-header=x-scope-orgid", "--traces.tempo.endpoint=http://localhost:3200", + "--traces.write-timeout=5s", "--rbac.config=/etc/tempo-gateway/rbac/rbac.yaml", "--tenants.config=/etc/tempo-gateway/tenants/tenants.yaml", "--log.level=info", diff --git a/internal/manifests/oauthproxy/oauth_proxy.go b/internal/manifests/oauthproxy/oauth_proxy.go index 2e90f2f12..f440e9c5c 100644 --- a/internal/manifests/oauthproxy/oauth_proxy.go +++ b/internal/manifests/oauthproxy/oauth_proxy.go @@ -3,6 +3,7 @@ package oauthproxy import ( "fmt" "strings" + "time" routev1 "github.com/openshift/api/route/v1" "github.com/operator-framework/operator-lib/proxy" @@ -69,9 +70,12 @@ func PatchRouteForOauthProxy(route *routev1.Route) { // point route to the oauth } // PatchStatefulSetForOauthProxy returns a modified StatefulSet with the oauth sidecar container and the right service account. -func PatchStatefulSetForOauthProxy(tempo metav1.ObjectMeta, +func PatchStatefulSetForOauthProxy( + tempo metav1.ObjectMeta, authSpec *v1alpha1.JaegerQueryAuthenticationSpec, - config configv1alpha1.ProjectConfig, statefulSet *v1.StatefulSet) { + timeout time.Duration, + config configv1alpha1.ProjectConfig, + statefulSet *v1.StatefulSet) { statefulSet.Spec.Template.Spec.Volumes = append(statefulSet.Spec.Template.Spec.Volumes, corev1.Volume{ Name: getTLSSecretNameForFrontendService(tempo.Name), VolumeSource: corev1.VolumeSource{ @@ -82,7 +86,7 @@ func PatchStatefulSetForOauthProxy(tempo metav1.ObjectMeta, }) statefulSet.Spec.Template.Spec.Containers = append(statefulSet.Spec.Template.Spec.Containers, - oAuthProxyContainer(tempo.Name, statefulSet.Spec.Template.Spec.ServiceAccountName, authSpec, config.DefaultImages.OauthProxy)) + oAuthProxyContainer(tempo.Name, statefulSet.Spec.Template.Spec.ServiceAccountName, authSpec, timeout, config.DefaultImages.OauthProxy)) } // PatchDeploymentForOauthProxy returns a modified deployment with the oauth sidecar container and the right service account. @@ -90,6 +94,7 @@ func PatchDeploymentForOauthProxy( tempo metav1.ObjectMeta, config configv1alpha1.ProjectConfig, authSpec *v1alpha1.JaegerQueryAuthenticationSpec, + timeout time.Duration, imageSpec configv1alpha1.ImagesSpec, dep *v1.Deployment) { dep.Spec.Template.Spec.Volumes = append(dep.Spec.Template.Spec.Volumes, corev1.Volume{ @@ -109,15 +114,18 @@ func PatchDeploymentForOauthProxy( } dep.Spec.Template.Spec.Containers = append(dep.Spec.Template.Spec.Containers, - oAuthProxyContainer(tempo.Name, naming.Name(manifestutils.QueryFrontendComponentName, tempo.Name), - authSpec, oauthProxyImage)) + oAuthProxyContainer(tempo.Name, + naming.Name(manifestutils.QueryFrontendComponentName, tempo.Name), + authSpec, + timeout, + oauthProxyImage)) } func getTLSSecretNameForFrontendService(tempoName string) string { return fmt.Sprintf("%s-ui-oauth-proxy-tls", tempoName) } -func proxyInitArguments(serviceAccountName string) []string { +func proxyInitArguments(serviceAccountName string, timeout time.Duration) []string { return []string{ // The SA Token is injected by admission controller by adding a volume via pod mutation // In Kubernetes 1.24 the SA token is short-lived (default 1h) @@ -136,6 +144,7 @@ func proxyInitArguments(serviceAccountName string) []string { fmt.Sprintf("--tls-cert=%s/tls.crt", tlsProxyPath), fmt.Sprintf("--tls-key=%s/tls.key", tlsProxyPath), fmt.Sprintf("--upstream=http://localhost:%d", manifestutils.PortJaegerUI), + fmt.Sprintf("--upstream-timeout=%s", timeout.String()), } } @@ -143,9 +152,10 @@ func oAuthProxyContainer( tempo string, serviceAccountName string, authSpec *v1alpha1.JaegerQueryAuthenticationSpec, + timeout time.Duration, oauthProxyImage string, ) corev1.Container { - args := proxyInitArguments(serviceAccountName) + args := proxyInitArguments(serviceAccountName, timeout) if len(strings.TrimSpace(authSpec.SAR)) > 0 { args = append(args, fmt.Sprintf("--openshift-sar=%s", authSpec.SAR)) diff --git a/internal/manifests/oauthproxy/oauth_proxy_test.go b/internal/manifests/oauthproxy/oauth_proxy_test.go index bd09a96fe..bde7ff220 100644 --- a/internal/manifests/oauthproxy/oauth_proxy_test.go +++ b/internal/manifests/oauthproxy/oauth_proxy_test.go @@ -2,6 +2,7 @@ package oauthproxy import ( "fmt" + "time" "testing" @@ -45,6 +46,7 @@ func TestOauthProxyContainer(t *testing.T) { fmt.Sprintf("--tls-cert=%s/tls.crt", tlsProxyPath), fmt.Sprintf("--tls-key=%s/tls.key", tlsProxyPath), fmt.Sprintf("--upstream=http://localhost:%d", manifestutils.PortJaegerUI), + "--upstream-timeout=5s", }, tempo: v1alpha1.TempoStack{ ObjectMeta: metav1.ObjectMeta{ @@ -75,6 +77,7 @@ func TestOauthProxyContainer(t *testing.T) { fmt.Sprintf("--tls-cert=%s/tls.crt", tlsProxyPath), fmt.Sprintf("--tls-key=%s/tls.key", tlsProxyPath), fmt.Sprintf("--upstream=http://localhost:%d", manifestutils.PortJaegerUI), + "--upstream-timeout=5s", "--openshift-sar={\"namespace\":\"app-dev\",\"resource\":\"services\",\"resourceName\":\"proxy\",\"verb\":\"get\"}", }, tempo: v1alpha1.TempoStack{ @@ -112,6 +115,7 @@ func TestOauthProxyContainer(t *testing.T) { container := oAuthProxyContainer(params.Tempo.Name, naming.Name(manifestutils.QueryFrontendComponentName, params.Tempo.Name), params.Tempo.Spec.Template.QueryFrontend.JaegerQuery.Authentication, + time.Second*5, customImage, ) expected := corev1.Container{ @@ -340,6 +344,7 @@ func TestPatchDeploymentForOauthProxy(t *testing.T) { params.Tempo.ObjectMeta, params.CtrlConfig, params.Tempo.Spec.Template.QueryFrontend.JaegerQuery.Authentication, + time.Second*5, params.Tempo.Spec.Images, dep) @@ -478,6 +483,7 @@ func TestPatchStatefulSetForOauthProxy(t *testing.T) { PatchStatefulSetForOauthProxy( params.Tempo.ObjectMeta, params.Tempo.Spec.Template.QueryFrontend.JaegerQuery.Authentication, + time.Second*5, params.CtrlConfig, statefulSet) diff --git a/internal/manifests/queryfrontend/query_frontend.go b/internal/manifests/queryfrontend/query_frontend.go index f457135df..bb232db0e 100644 --- a/internal/manifests/queryfrontend/query_frontend.go +++ b/internal/manifests/queryfrontend/query_frontend.go @@ -34,6 +34,8 @@ const ( containerNameTempo = "tempo" containerNameJaegerQuery = "jaeger-query" containerNameTempoQuery = "tempo-query" + + timeoutRouteAnnotation = "haproxy.router.openshift.io/timeout" ) // BuildQueryFrontend creates the query-frontend objects. @@ -85,9 +87,12 @@ func BuildQueryFrontend(params manifestutils.Params) ([]client.Object, error) { if jaegerUIAuthentication != nil && jaegerUIAuthentication.Enabled { oauthproxy.PatchDeploymentForOauthProxy( - tempo.ObjectMeta, params.CtrlConfig, + tempo.ObjectMeta, + params.CtrlConfig, tempo.Spec.Template.QueryFrontend.JaegerQuery.Authentication, - tempo.Spec.Images, d) + tempo.Spec.Timeout.Duration, + tempo.Spec.Images, + d) oauthproxy.PatchQueryFrontEndService(getQueryFrontendService(tempo, svcs), tempo.Name) manifests = append(manifests, oauthproxy.OAuthServiceAccount(params)) @@ -582,12 +587,20 @@ func route(tempo v1alpha1.TempoStack) (*routev1.Route, error) { serviceName := naming.Name(manifestutils.QueryFrontendComponentName, tempo.Name) + annotations := tempo.Spec.Template.QueryFrontend.JaegerQuery.Ingress.Annotations + if annotations == nil { + annotations = map[string]string{} + } + if annotations[timeoutRouteAnnotation] == "" { + annotations[timeoutRouteAnnotation] = fmt.Sprintf("%ds", int(tempo.Spec.Timeout.Duration.Seconds())) + } + return &routev1.Route{ ObjectMeta: metav1.ObjectMeta{ Name: queryFrontendName, Namespace: tempo.Namespace, Labels: labels, - Annotations: tempo.Spec.Template.QueryFrontend.JaegerQuery.Ingress.Annotations, + Annotations: annotations, }, Spec: routev1.RouteSpec{ Host: tempo.Spec.Template.QueryFrontend.JaegerQuery.Ingress.Host, diff --git a/internal/manifests/queryfrontend/query_frontend_test.go b/internal/manifests/queryfrontend/query_frontend_test.go index c11c7f352..e9a5650f9 100644 --- a/internal/manifests/queryfrontend/query_frontend_test.go +++ b/internal/manifests/queryfrontend/query_frontend_test.go @@ -511,6 +511,9 @@ func TestQueryFrontendJaegerRoute(t *testing.T) { Name: naming.Name(manifestutils.QueryFrontendComponentName, "test"), Namespace: "project1", Labels: manifestutils.ComponentLabels("query-frontend", "test"), + Annotations: map[string]string{ + "haproxy.router.openshift.io/timeout": "0s", + }, }, Spec: routev1.RouteSpec{ To: routev1.RouteTargetReference{ @@ -810,6 +813,9 @@ func TestQueryFrontendJaegerRouteSecured(t *testing.T) { Name: naming.Name(manifestutils.QueryFrontendComponentName, "test"), Namespace: "project1", Labels: manifestutils.ComponentLabels("query-frontend", "test"), + Annotations: map[string]string{ + "haproxy.router.openshift.io/timeout": "0s", + }, }, Spec: routev1.RouteSpec{ To: routev1.RouteTargetReference{ diff --git a/internal/webhooks/tempostack_webhook.go b/internal/webhooks/tempostack_webhook.go index 7f6a1f3e1..ea0f944fa 100644 --- a/internal/webhooks/tempostack_webhook.go +++ b/internal/webhooks/tempostack_webhook.go @@ -33,6 +33,7 @@ var ( zeroQuantity = resource.MustParse("0Gi") tenGBQuantity = resource.MustParse("10Gi") defaultServicesDuration = metav1.Duration{Duration: time.Hour * 24 * 3} + defaultTimeout = metav1.Duration{Duration: time.Second * 30} ) // TempoStackWebhook provides webhooks for TempoStack CR. @@ -171,6 +172,10 @@ func (d *Defaulter) Default(ctx context.Context, obj runtime.Object) error { } } + if r.Spec.Timeout.Duration == 0 { + r.Spec.Timeout = defaultTimeout + } + return nil } diff --git a/internal/webhooks/tempostack_webhook_test.go b/internal/webhooks/tempostack_webhook_test.go index b051ab134..b24f02553 100644 --- a/internal/webhooks/tempostack_webhook_test.go +++ b/internal/webhooks/tempostack_webhook_test.go @@ -86,6 +86,7 @@ func TestDefault(t *testing.T) { }, Spec: v1alpha1.TempoStackSpec{ ReplicationFactor: 2, + Timeout: metav1.Duration{Duration: time.Second * 30}, Images: configv1alpha1.ImagesSpec{ Tempo: "docker.io/grafana/tempo:1.2.3", TempoQuery: "docker.io/grafana/tempo-query:1.2.3", @@ -162,6 +163,7 @@ func TestDefault(t *testing.T) { }, Spec: v1alpha1.TempoStackSpec{ ReplicationFactor: 1, + Timeout: metav1.Duration{Duration: time.Second * 30}, Images: configv1alpha1.ImagesSpec{}, ServiceAccount: "tempo-test", Retention: v1alpha1.RetentionSpec{ @@ -244,6 +246,7 @@ func TestDefault(t *testing.T) { }, Spec: v1alpha1.TempoStackSpec{ ReplicationFactor: 1, + Timeout: metav1.Duration{Duration: time.Second * 30}, Images: configv1alpha1.ImagesSpec{}, ServiceAccount: "tempo-test", Retention: v1alpha1.RetentionSpec{ @@ -326,6 +329,7 @@ func TestDefault(t *testing.T) { }, Spec: v1alpha1.TempoStackSpec{ ReplicationFactor: 1, + Timeout: metav1.Duration{Duration: time.Second * 30}, Images: configv1alpha1.ImagesSpec{}, ServiceAccount: "tempo-test", Retention: v1alpha1.RetentionSpec{ @@ -425,6 +429,7 @@ func TestDefault(t *testing.T) { }, Spec: v1alpha1.TempoStackSpec{ ReplicationFactor: 1, + Timeout: metav1.Duration{Duration: time.Second * 30}, Images: configv1alpha1.ImagesSpec{}, ServiceAccount: "tempo-test", Retention: v1alpha1.RetentionSpec{ @@ -491,6 +496,108 @@ func TestDefault(t *testing.T) { Distribution: "upstream", }, }, + { + name: "timeout is set", + input: &v1alpha1.TempoStack{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: v1alpha1.TempoStackSpec{ + ReplicationFactor: 2, + Images: configv1alpha1.ImagesSpec{ + Tempo: "docker.io/grafana/tempo:1.2.3", + TempoQuery: "docker.io/grafana/tempo-query:1.2.3", + TempoGateway: "docker.io/observatorium/gateway:1.2.3", + TempoGatewayOpa: "docker.io/observatorium/opa-openshift:1.2.4", + OauthProxy: "docker.io/observatorium/oauth-proxy:1.2.3", + }, + ServiceAccount: "tempo-test", + Retention: v1alpha1.RetentionSpec{ + Global: v1alpha1.RetentionConfig{ + Traces: metav1.Duration{Duration: time.Hour}, + }, + }, + Timeout: metav1.Duration{Duration: time.Hour}, + StorageSize: resource.MustParse("1Gi"), + LimitSpec: v1alpha1.LimitSpec{ + Global: v1alpha1.RateLimitSpec{ + Query: v1alpha1.QueryLimit{ + MaxSearchDuration: metav1.Duration{Duration: 1 * time.Hour}, + }, + }, + }, + }, + }, + expected: &v1alpha1.TempoStack{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Labels: map[string]string{ + "app.kubernetes.io/managed-by": "tempo-operator", + "tempo.grafana.com/distribution": "upstream", + }, + }, + Spec: v1alpha1.TempoStackSpec{ + ReplicationFactor: 2, + Timeout: metav1.Duration{Duration: time.Hour}, + Images: configv1alpha1.ImagesSpec{ + Tempo: "docker.io/grafana/tempo:1.2.3", + TempoQuery: "docker.io/grafana/tempo-query:1.2.3", + TempoGateway: "docker.io/observatorium/gateway:1.2.3", + TempoGatewayOpa: "docker.io/observatorium/opa-openshift:1.2.4", + OauthProxy: "docker.io/observatorium/oauth-proxy:1.2.3", + }, + ServiceAccount: "tempo-test", + Retention: v1alpha1.RetentionSpec{ + Global: v1alpha1.RetentionConfig{ + Traces: metav1.Duration{Duration: time.Hour}, + }, + }, + StorageSize: resource.MustParse("1Gi"), + LimitSpec: v1alpha1.LimitSpec{ + Global: v1alpha1.RateLimitSpec{ + Query: v1alpha1.QueryLimit{ + MaxSearchDuration: metav1.Duration{Duration: 1 * time.Hour}, + }, + }, + }, + SearchSpec: v1alpha1.SearchSpec{ + MaxDuration: metav1.Duration{Duration: 0}, + DefaultResultLimit: &defaultDefaultResultLimit, + }, + Template: v1alpha1.TempoTemplateSpec{ + Compactor: v1alpha1.TempoComponentSpec{ + Replicas: ptr.To(int32(1)), + }, + Distributor: v1alpha1.TempoDistributorSpec{ + TempoComponentSpec: v1alpha1.TempoComponentSpec{ + Replicas: ptr.To(int32(1)), + }, + TLS: v1alpha1.TLSSpec{}, + }, + Ingester: v1alpha1.TempoComponentSpec{ + Replicas: ptr.To(int32(1)), + }, + Querier: v1alpha1.TempoComponentSpec{ + Replicas: ptr.To(int32(1)), + }, + Gateway: v1alpha1.TempoGatewaySpec{ + TempoComponentSpec: v1alpha1.TempoComponentSpec{ + Replicas: ptr.To(int32(1)), + }, + }, + QueryFrontend: v1alpha1.TempoQueryFrontendSpec{ + TempoComponentSpec: v1alpha1.TempoComponentSpec{ + Replicas: ptr.To(int32(1)), + }, + JaegerQuery: v1alpha1.JaegerQuerySpec{ + ServicesQueryDuration: &defaultServicesDuration, + }, + }, + }, + }, + }, + ctrlConfig: defaultCfgConfig, + }, } for _, test := range tests { diff --git a/tests/e2e-openshift/component-replicas/install-tempo-assert.yaml b/tests/e2e-openshift/component-replicas/install-tempo-assert.yaml index 450abdec7..d306e7429 100644 --- a/tests/e2e-openshift/component-replicas/install-tempo-assert.yaml +++ b/tests/e2e-openshift/component-replicas/install-tempo-assert.yaml @@ -156,6 +156,7 @@ spec: - --web.internal.listen=0.0.0.0:8081 - --traces.write.otlpgrpc.endpoint=tempo-cmpreps-distributor.chainsaw-replicas.svc.cluster.local:4317 - --traces.write.otlphttp.endpoint=https://tempo-cmpreps-distributor.chainsaw-replicas.svc.cluster.local:4318 + - --traces.write-timeout=30s - --traces.tempo.endpoint=https://tempo-cmpreps-query-frontend.chainsaw-replicas.svc.cluster.local:3200 - --grpc.listen=0.0.0.0:8090 - --rbac.config=/etc/tempo-gateway/cm/rbac.yaml diff --git a/tests/e2e-openshift/component-replicas/scale-tempo-assert.yaml b/tests/e2e-openshift/component-replicas/scale-tempo-assert.yaml index c23c4bb52..2d07ffb5d 100644 --- a/tests/e2e-openshift/component-replicas/scale-tempo-assert.yaml +++ b/tests/e2e-openshift/component-replicas/scale-tempo-assert.yaml @@ -157,6 +157,7 @@ spec: - --traces.write.otlpgrpc.endpoint=tempo-cmpreps-distributor.chainsaw-replicas.svc.cluster.local:4317 - --traces.write.otlphttp.endpoint=https://tempo-cmpreps-distributor.chainsaw-replicas.svc.cluster.local:4318 - --traces.tempo.endpoint=https://tempo-cmpreps-query-frontend.chainsaw-replicas.svc.cluster.local:3200 + - --traces.write-timeout=30s - --grpc.listen=0.0.0.0:8090 - --rbac.config=/etc/tempo-gateway/cm/rbac.yaml - --tenants.config=/etc/tempo-gateway/secret/tenants.yaml diff --git a/tests/e2e-openshift/monolithic-route/install-tempo-assert.yaml b/tests/e2e-openshift/monolithic-route/install-tempo-assert.yaml index 241609101..e27f27322 100644 --- a/tests/e2e-openshift/monolithic-route/install-tempo-assert.yaml +++ b/tests/e2e-openshift/monolithic-route/install-tempo-assert.yaml @@ -130,6 +130,7 @@ spec: - --tls-cert=/etc/tls/private/tls.crt - --tls-key=/etc/tls/private/tls.key - --upstream=http://localhost:16686 + - --upstream-timeout=70s - '--openshift-sar={"namespace": "chainsaw-mono-route", "resource": "pods", "verb": "get"}' name: oauth-proxy @@ -248,3 +249,24 @@ spec: app.kubernetes.io/instance: mono-route app.kubernetes.io/managed-by: tempo-operator app.kubernetes.io/name: tempo-monolithic +--- +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + labels: + app.kubernetes.io/component: jaegerui + app.kubernetes.io/instance: mono-route + app.kubernetes.io/managed-by: tempo-operator + app.kubernetes.io/name: tempo-monolithic + name: tempo-mono-jaegerui + namespace: chainsaw-mono-route + annotations: + haproxy.router.openshift.io/timeout: 70s +spec: + port: + targetPort: oauth-proxy + tls: + termination: reencrypt + to: + kind: Service + name: tempo-mono-jaegerui diff --git a/tests/e2e-openshift/monolithic-route/install-tempo.yaml b/tests/e2e-openshift/monolithic-route/install-tempo.yaml index ca7c389c4..c44f9bdeb 100644 --- a/tests/e2e-openshift/monolithic-route/install-tempo.yaml +++ b/tests/e2e-openshift/monolithic-route/install-tempo.yaml @@ -4,6 +4,7 @@ metadata: name: mono-route namespace: chainsaw-mono-route spec: + timeout: 70s jaegerui: enabled: true route: diff --git a/tests/e2e-openshift/monolithic-single-tenant-auth/install-tempo-assert.yaml b/tests/e2e-openshift/monolithic-single-tenant-auth/install-tempo-assert.yaml index 2ddab9151..d32052933 100644 --- a/tests/e2e-openshift/monolithic-single-tenant-auth/install-tempo-assert.yaml +++ b/tests/e2e-openshift/monolithic-single-tenant-auth/install-tempo-assert.yaml @@ -130,6 +130,7 @@ spec: - --tls-cert=/etc/tls/private/tls.crt - --tls-key=/etc/tls/private/tls.key - --upstream=http://localhost:16686 + - --upstream-timeout=30s - '--openshift-sar={"namespace": "chainsaw-mst", "resource": "pods", "verb": "get"}' name: oauth-proxy diff --git a/tests/e2e-openshift/multitenancy/01-assert.yaml b/tests/e2e-openshift/multitenancy/01-assert.yaml index cb1017286..22581acd0 100644 --- a/tests/e2e-openshift/multitenancy/01-assert.yaml +++ b/tests/e2e-openshift/multitenancy/01-assert.yaml @@ -151,6 +151,7 @@ spec: - --web.internal.listen=0.0.0.0:8081 - --traces.write.otlpgrpc.endpoint=tempo-simplest-distributor.chainsaw-multitenancy.svc.cluster.local:4317 - --traces.write.otlphttp.endpoint=https://tempo-simplest-distributor.chainsaw-multitenancy.svc.cluster.local:4318 + - --traces.write-timeout=30s - --traces.tempo.endpoint=https://tempo-simplest-query-frontend.chainsaw-multitenancy.svc.cluster.local:3200 - --grpc.listen=0.0.0.0:8090 - --rbac.config=/etc/tempo-gateway/cm/rbac.yaml diff --git a/tests/e2e-openshift/route/install-tempo-assert.yaml b/tests/e2e-openshift/route/install-tempo-assert.yaml index 78e16793f..57b4ad8e6 100644 --- a/tests/e2e-openshift/route/install-tempo-assert.yaml +++ b/tests/e2e-openshift/route/install-tempo-assert.yaml @@ -3,6 +3,7 @@ kind: Route metadata: annotations: example_annotation: example_value + haproxy.router.openshift.io/timeout: 30s labels: app.kubernetes.io/component: query-frontend app.kubernetes.io/instance: simplest diff --git a/tests/e2e-openshift/route/install-tempo.yaml b/tests/e2e-openshift/route/install-tempo.yaml index 0feb8af11..c51ea0aeb 100644 --- a/tests/e2e-openshift/route/install-tempo.yaml +++ b/tests/e2e-openshift/route/install-tempo.yaml @@ -16,6 +16,7 @@ kind: TempoStack metadata: name: simplest spec: + timeout: 70s storage: secret: name: minio-test diff --git a/tests/e2e-openshift/tls-monolithic-singletenant/01-assert.yaml b/tests/e2e-openshift/tls-monolithic-singletenant/01-assert.yaml index 0b1f9dd38..5be1ff02b 100644 --- a/tests/e2e-openshift/tls-monolithic-singletenant/01-assert.yaml +++ b/tests/e2e-openshift/tls-monolithic-singletenant/01-assert.yaml @@ -136,6 +136,7 @@ spec: - --tls-cert=/etc/tls/private/tls.crt - --tls-key=/etc/tls/private/tls.key - --upstream=http://localhost:16686 + - --upstream-timeout=30s - '--openshift-sar={"namespace": "chainsaw-tls-mono-st", "resource": "pods", "verb": "get"}' name: oauth-proxy diff --git a/tests/e2e/monolithic-extraconfig/install-tempo-assert.yaml b/tests/e2e/monolithic-extraconfig/install-tempo-assert.yaml index d9897a059..b2244e1df 100644 --- a/tests/e2e/monolithic-extraconfig/install-tempo-assert.yaml +++ b/tests/e2e/monolithic-extraconfig/install-tempo-assert.yaml @@ -98,6 +98,8 @@ data: max_retries: 3 server: http_listen_port: 3200 + http_server_read_timeout: 30s + http_server_write_timeout: 30s storage: trace: backend: local diff --git a/tests/e2e/tempostack-extraconfig/install-tempostack-assert.yaml b/tests/e2e/tempostack-extraconfig/install-tempostack-assert.yaml index aa23663bf..4cad151d3 100644 --- a/tests/e2e/tempostack-extraconfig/install-tempostack-assert.yaml +++ b/tests/e2e/tempostack-extraconfig/install-tempostack-assert.yaml @@ -456,8 +456,8 @@ data: client_ca_file: /var/run/ca/service-ca.crt key_file: /var/run/tls/server/tls.key http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 10m + http_server_write_timeout: 10m log_format: logfmt tls_min_version: VersionTLS13 storage: @@ -568,8 +568,8 @@ data: client_ca_file: /var/run/ca/service-ca.crt key_file: /var/run/tls/server/tls.key http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 10m + http_server_write_timeout: 10m http_tls_config: cert_file: /var/run/tls/server/tls.crt client_auth_type: RequireAndVerifyClientCert diff --git a/tests/e2e/tempostack-extraconfig/install-tempostack.yaml b/tests/e2e/tempostack-extraconfig/install-tempostack.yaml index 812ff0bc3..2de1a58a6 100644 --- a/tests/e2e/tempostack-extraconfig/install-tempostack.yaml +++ b/tests/e2e/tempostack-extraconfig/install-tempostack.yaml @@ -3,8 +3,12 @@ kind: TempoStack metadata: name: simplest spec: + timeout: 70s extraConfig: tempo: + server: + http_server_write_timeout: 10m + http_server_read_timeout: 10m querier: search: query_timeout: 180s From ff0d0dcb676941d1c9e2f4250614224f544e1512 Mon Sep 17 00:00:00 2001 From: Ishwar Kanse Date: Tue, 8 Oct 2024 15:29:27 +0530 Subject: [PATCH 04/10] Test timeout for non-multitenant TempoStack and TempoMonolithic instances (#1049) --- .../monolithic-route/chainsaw-test.yaml | 1 - .../install-tempo-assert.yaml | 54 ++- .../monolithic-route/install-tempo.yaml | 2 +- tests/e2e-openshift/route/chainsaw-test.yaml | 12 + .../route/install-storage-assert.yaml | 6 + .../e2e-openshift/route/install-storage.yaml | 75 ++++ .../route/install-tempo-assert.yaml | 336 +++++++++++++++++- tests/e2e-openshift/route/install-tempo.yaml | 4 +- .../01-assert.yaml | 51 +++ .../tls-singletenant/01-assert.yaml | 140 +++++++- .../tls-singletenant/01-install-tempo.yaml | 3 +- 11 files changed, 667 insertions(+), 17 deletions(-) create mode 100644 tests/e2e-openshift/route/install-storage-assert.yaml create mode 100644 tests/e2e-openshift/route/install-storage.yaml diff --git a/tests/e2e-openshift/monolithic-route/chainsaw-test.yaml b/tests/e2e-openshift/monolithic-route/chainsaw-test.yaml index aea3ab58b..2dac7e15c 100755 --- a/tests/e2e-openshift/monolithic-route/chainsaw-test.yaml +++ b/tests/e2e-openshift/monolithic-route/chainsaw-test.yaml @@ -1,4 +1,3 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json apiVersion: chainsaw.kyverno.io/v1alpha1 kind: Test metadata: diff --git a/tests/e2e-openshift/monolithic-route/install-tempo-assert.yaml b/tests/e2e-openshift/monolithic-route/install-tempo-assert.yaml index e27f27322..07874955c 100644 --- a/tests/e2e-openshift/monolithic-route/install-tempo-assert.yaml +++ b/tests/e2e-openshift/monolithic-route/install-tempo-assert.yaml @@ -130,7 +130,7 @@ spec: - --tls-cert=/etc/tls/private/tls.crt - --tls-key=/etc/tls/private/tls.key - --upstream=http://localhost:16686 - - --upstream-timeout=70s + - --upstream-timeout=2m0s - '--openshift-sar={"namespace": "chainsaw-mono-route", "resource": "pods", "verb": "get"}' name: oauth-proxy @@ -249,19 +249,59 @@ spec: app.kubernetes.io/instance: mono-route app.kubernetes.io/managed-by: tempo-operator app.kubernetes.io/name: tempo-monolithic + +--- +apiVersion: v1 +data: + tempo-query.yaml: | + address: 127.0.0.1:7777 + backend: 127.0.0.1:3200 + tenant_header_key: x-scope-orgid + services_query_duration: 72h0m0s + tempo.yaml: | + server: + http_listen_port: 3200 + http_server_read_timeout: 2m0s + http_server_write_timeout: 2m0s + internal_server: + enable: true + http_listen_address: 0.0.0.0 + storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/blocks + distributor: + receivers: + otlp: + protocols: + grpc: {} + http: {} + usage_report: + reporting_enabled: false +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/component: config + app.kubernetes.io/instance: mono-route + app.kubernetes.io/managed-by: tempo-operator + app.kubernetes.io/name: tempo-monolithic + name: tempo-mono-route-config + --- apiVersion: route.openshift.io/v1 kind: Route metadata: + annotations: + haproxy.router.openshift.io/timeout: 120s labels: app.kubernetes.io/component: jaegerui app.kubernetes.io/instance: mono-route app.kubernetes.io/managed-by: tempo-operator app.kubernetes.io/name: tempo-monolithic - name: tempo-mono-jaegerui - namespace: chainsaw-mono-route - annotations: - haproxy.router.openshift.io/timeout: 70s + name: tempo-mono-route-jaegerui spec: port: targetPort: oauth-proxy @@ -269,4 +309,6 @@ spec: termination: reencrypt to: kind: Service - name: tempo-mono-jaegerui + name: tempo-mono-route-jaegerui + weight: 100 + wildcardPolicy: None diff --git a/tests/e2e-openshift/monolithic-route/install-tempo.yaml b/tests/e2e-openshift/monolithic-route/install-tempo.yaml index c44f9bdeb..860959934 100644 --- a/tests/e2e-openshift/monolithic-route/install-tempo.yaml +++ b/tests/e2e-openshift/monolithic-route/install-tempo.yaml @@ -4,7 +4,7 @@ metadata: name: mono-route namespace: chainsaw-mono-route spec: - timeout: 70s + timeout: 2m jaegerui: enabled: true route: diff --git a/tests/e2e-openshift/route/chainsaw-test.yaml b/tests/e2e-openshift/route/chainsaw-test.yaml index a8751c4b0..a3ab3f7e7 100755 --- a/tests/e2e-openshift/route/chainsaw-test.yaml +++ b/tests/e2e-openshift/route/chainsaw-test.yaml @@ -4,10 +4,22 @@ kind: Test metadata: name: route spec: + namespace: chainsaw-route steps: + - name: Install Minio storage + try: + - apply: + file: install-storage.yaml + - assert: + file: install-storage-assert.yaml - name: Install TempoStack with ingress type route try: - apply: file: install-tempo.yaml - assert: file: install-tempo-assert.yaml + - name: Check the status of TempoStack + try: + - script: + timeout: 5m + content: kubectl get --namespace $NAMESPACE tempo simplest -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' | grep True diff --git a/tests/e2e-openshift/route/install-storage-assert.yaml b/tests/e2e-openshift/route/install-storage-assert.yaml new file mode 100644 index 000000000..afa1f9c34 --- /dev/null +++ b/tests/e2e-openshift/route/install-storage-assert.yaml @@ -0,0 +1,6 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: minio +status: + readyReplicas: 1 diff --git a/tests/e2e-openshift/route/install-storage.yaml b/tests/e2e-openshift/route/install-storage.yaml new file mode 100644 index 000000000..d85862e72 --- /dev/null +++ b/tests/e2e-openshift/route/install-storage.yaml @@ -0,0 +1,75 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + labels: + app.kubernetes.io/name: minio + name: minio +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: minio +spec: + selector: + matchLabels: + app.kubernetes.io/name: minio + strategy: + type: Recreate + template: + metadata: + labels: + app.kubernetes.io/name: minio + spec: + containers: + - command: + - /bin/sh + - -c + - | + mkdir -p /storage/tempo && \ + minio server /storage + env: + - name: MINIO_ACCESS_KEY + value: tempo + - name: MINIO_SECRET_KEY + value: supersecret + image: quay.io/minio/minio:latest + name: minio + ports: + - containerPort: 9000 + volumeMounts: + - mountPath: /storage + name: storage + volumes: + - name: storage + persistentVolumeClaim: + claimName: minio +--- +apiVersion: v1 +kind: Service +metadata: + name: minio +spec: + ports: + - port: 9000 + protocol: TCP + targetPort: 9000 + selector: + app.kubernetes.io/name: minio + type: ClusterIP +--- +apiVersion: v1 +kind: Secret +metadata: + name: minio +stringData: + endpoint: http://minio:9000 + bucket: tempo + access_key_id: tempo + access_key_secret: supersecret +type: Opaque diff --git a/tests/e2e-openshift/route/install-tempo-assert.yaml b/tests/e2e-openshift/route/install-tempo-assert.yaml index 57b4ad8e6..8fba41e72 100644 --- a/tests/e2e-openshift/route/install-tempo-assert.yaml +++ b/tests/e2e-openshift/route/install-tempo-assert.yaml @@ -3,7 +3,7 @@ kind: Route metadata: annotations: example_annotation: example_value - haproxy.router.openshift.io/timeout: 30s + haproxy.router.openshift.io/timeout: 120s labels: app.kubernetes.io/component: query-frontend app.kubernetes.io/instance: simplest @@ -20,3 +20,337 @@ spec: kind: Service name: tempo-simplest-query-frontend weight: 100 + +--- +apiVersion: v1 +data: + overrides.yaml: | + overrides: + tempo-query-frontend.yaml: | + compactor: + compaction: + block_retention: 48h0m0s + ring: + kvstore: + store: memberlist + distributor: + receivers: + jaeger: + protocols: + thrift_http: + endpoint: 0.0.0.0:14268 + thrift_binary: + endpoint: 0.0.0.0:6832 + thrift_compact: + endpoint: 0.0.0.0:6831 + grpc: + endpoint: 0.0.0.0:14250 + zipkin: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + ring: + kvstore: + store: memberlist + ingester: + lifecycler: + ring: + kvstore: + store: memberlist + replication_factor: 1 + tokens_file_path: /var/tempo/tokens.json + max_block_duration: 10m + memberlist: + abort_if_cluster_join_fails: false + join_members: + - tempo-simplest-gossip-ring + multitenancy_enabled: false + querier: + max_concurrent_queries: 20 + frontend_worker: + frontend_address: tempo-simplest-query-frontend-discovery:9095 + grpc_client_config: + tls_enabled: true + tls_cert_path: /var/run/tls/server/tls.crt + tls_key_path: /var/run/tls/server/tls.key + tls_ca_path: /var/run/ca/service-ca.crt + tls_server_name: tempo-simplest-query-frontend.chainsaw-route.svc.cluster.local + tls_min_version: VersionTLS13 + search: + external_hedge_requests_at: 8s + external_hedge_requests_up_to: 2 + server: + grpc_server_max_recv_msg_size: 4194304 + grpc_server_max_send_msg_size: 4194304 + http_listen_port: 3200 + http_server_read_timeout: 2m0s + http_server_write_timeout: 2m0s + log_format: logfmt + tls_min_version: VersionTLS13 + grpc_tls_config: + cert_file: /var/run/tls/server/tls.crt + key_file: /var/run/tls/server/tls.key + client_ca_file: /var/run/ca/service-ca.crt + client_auth_type: RequireAndVerifyClientCert + storage: + trace: + backend: s3 + blocklist_poll: 5m + cache: none + s3: + endpoint: minio:9000 + bucket: tempo + insecure: true + local: + path: /var/tempo/traces + wal: + path: /var/tempo/wal + usage_report: + reporting_enabled: false + query_frontend: + search: + concurrent_jobs: 2000 + max_duration: 0s + default_result_limit: 20 + ingester_client: + grpc_client_config: + tls_enabled: true + tls_cert_path: /var/run/tls/server/tls.crt + tls_key_path: /var/run/tls/server/tls.key + tls_ca_path: /var/run/ca/service-ca.crt + tls_server_name: tempo-simplest-ingester.chainsaw-route.svc.cluster.local + tls_insecure_skip_verify: false + tls_min_version: VersionTLS13 + tempo-query.yaml: | + address: 127.0.0.1:7777 + backend: 127.0.0.1:3200 + tenant_header_key: x-scope-orgid + services_query_duration: 72h0m0s + tempo.yaml: | + compactor: + compaction: + block_retention: 48h0m0s + ring: + kvstore: + store: memberlist + distributor: + receivers: + jaeger: + protocols: + thrift_http: + endpoint: 0.0.0.0:14268 + thrift_binary: + endpoint: 0.0.0.0:6832 + thrift_compact: + endpoint: 0.0.0.0:6831 + grpc: + endpoint: 0.0.0.0:14250 + zipkin: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + ring: + kvstore: + store: memberlist + ingester: + lifecycler: + ring: + kvstore: + store: memberlist + replication_factor: 1 + tokens_file_path: /var/tempo/tokens.json + max_block_duration: 10m + memberlist: + abort_if_cluster_join_fails: false + join_members: + - tempo-simplest-gossip-ring + multitenancy_enabled: false + querier: + max_concurrent_queries: 20 + frontend_worker: + frontend_address: tempo-simplest-query-frontend-discovery:9095 + grpc_client_config: + tls_enabled: true + tls_cert_path: /var/run/tls/server/tls.crt + tls_key_path: /var/run/tls/server/tls.key + tls_ca_path: /var/run/ca/service-ca.crt + tls_server_name: tempo-simplest-query-frontend.chainsaw-route.svc.cluster.local + tls_min_version: VersionTLS13 + search: + external_hedge_requests_at: 8s + external_hedge_requests_up_to: 2 + internal_server: + enable: true + http_listen_address: "" + tls_min_version: VersionTLS13 + http_tls_config: + cert_file: /var/run/tls/server/tls.crt + key_file: /var/run/tls/server/tls.key + server: + grpc_server_max_recv_msg_size: 4194304 + grpc_server_max_send_msg_size: 4194304 + http_listen_port: 3200 + http_server_read_timeout: 2m0s + http_server_write_timeout: 2m0s + log_format: logfmt + tls_min_version: VersionTLS13 + grpc_tls_config: + cert_file: /var/run/tls/server/tls.crt + key_file: /var/run/tls/server/tls.key + client_ca_file: /var/run/ca/service-ca.crt + client_auth_type: RequireAndVerifyClientCert + http_tls_config: + cert_file: /var/run/tls/server/tls.crt + client_auth_type: RequireAndVerifyClientCert + key_file: /var/run/tls/server/tls.key + client_ca_file: /var/run/ca/service-ca.crt + storage: + trace: + backend: s3 + blocklist_poll: 5m + cache: none + s3: + endpoint: minio:9000 + bucket: tempo + insecure: true + local: + path: /var/tempo/traces + wal: + path: /var/tempo/wal + usage_report: + reporting_enabled: false + query_frontend: + search: + concurrent_jobs: 2000 + max_duration: 0s + default_result_limit: 20 + ingester_client: + grpc_client_config: + tls_enabled: true + tls_cert_path: /var/run/tls/server/tls.crt + tls_key_path: /var/run/tls/server/tls.key + tls_ca_path: /var/run/ca/service-ca.crt + tls_server_name: tempo-simplest-ingester.chainsaw-route.svc.cluster.local + tls_insecure_skip_verify: false + tls_min_version: VersionTLS13 +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/component: config + app.kubernetes.io/instance: simplest + app.kubernetes.io/managed-by: tempo-operator + app.kubernetes.io/name: tempo + name: tempo-simplest + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/component: compactor + app.kubernetes.io/instance: simplest + app.kubernetes.io/managed-by: tempo-operator + app.kubernetes.io/name: tempo + name: tempo-simplest-compactor +status: + availableReplicas: 1 + readyReplicas: 1 + replicas: 1 + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/component: distributor + app.kubernetes.io/instance: simplest + app.kubernetes.io/managed-by: tempo-operator + app.kubernetes.io/name: tempo + name: tempo-simplest-distributor +status: + availableReplicas: 1 + readyReplicas: 1 + replicas: 1 + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/component: querier + app.kubernetes.io/instance: simplest + app.kubernetes.io/managed-by: tempo-operator + app.kubernetes.io/name: tempo + name: tempo-simplest-querier +status: + availableReplicas: 1 + readyReplicas: 1 + replicas: 1 + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/component: query-frontend + app.kubernetes.io/instance: simplest + app.kubernetes.io/managed-by: tempo-operator + app.kubernetes.io/name: tempo + name: tempo-simplest-query-frontend +spec: + selector: + matchLabels: + app.kubernetes.io/component: query-frontend + app.kubernetes.io/instance: simplest + app.kubernetes.io/managed-by: tempo-operator + app.kubernetes.io/name: tempo + template: + metadata: + labels: + app.kubernetes.io/component: query-frontend + app.kubernetes.io/instance: simplest + app.kubernetes.io/managed-by: tempo-operator + app.kubernetes.io/name: tempo + tempo-gossip-member: "true" + spec: + containers: + - name: tempo + - name: jaeger-query + - name: tempo-query + - args: + - --cookie-secret-file=/var/run/secrets/kubernetes.io/serviceaccount/token + - --https-address=:8443 + - --openshift-service-account=tempo-simplest-query-frontend + - --provider=openshift + - --tls-cert=/etc/tls/private/tls.crt + - --tls-key=/etc/tls/private/tls.key + - --upstream=http://localhost:16686 + - --upstream-timeout=2m0s + - '--openshift-sar={"namespace": "chainsaw-route", "resource": "pods", "verb": "get"}' + name: oauth-proxy +status: + availableReplicas: 1 + readyReplicas: 1 + replicas: 1 + +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + app.kubernetes.io/component: ingester + app.kubernetes.io/instance: simplest + app.kubernetes.io/managed-by: tempo-operator + app.kubernetes.io/name: tempo + name: tempo-simplest-ingester +status: + availableReplicas: 1 + currentReplicas: 1 + readyReplicas: 1 + replicas: 1 \ No newline at end of file diff --git a/tests/e2e-openshift/route/install-tempo.yaml b/tests/e2e-openshift/route/install-tempo.yaml index c51ea0aeb..ac9556089 100644 --- a/tests/e2e-openshift/route/install-tempo.yaml +++ b/tests/e2e-openshift/route/install-tempo.yaml @@ -16,10 +16,10 @@ kind: TempoStack metadata: name: simplest spec: - timeout: 70s + timeout: 2m storage: secret: - name: minio-test + name: minio type: s3 storageSize: 200M template: diff --git a/tests/e2e-openshift/tls-monolithic-singletenant/01-assert.yaml b/tests/e2e-openshift/tls-monolithic-singletenant/01-assert.yaml index 5be1ff02b..3ea628016 100644 --- a/tests/e2e-openshift/tls-monolithic-singletenant/01-assert.yaml +++ b/tests/e2e-openshift/tls-monolithic-singletenant/01-assert.yaml @@ -276,6 +276,8 @@ spec: apiVersion: route.openshift.io/v1 kind: Route metadata: + annotations: + haproxy.router.openshift.io/timeout: 30s labels: app.kubernetes.io/component: jaegerui app.kubernetes.io/instance: mono @@ -291,3 +293,52 @@ spec: to: kind: Service name: tempo-mono-jaegerui + +--- +apiVersion: v1 +data: + tempo-query.yaml: | + address: 127.0.0.1:7777 + backend: 127.0.0.1:3200 + tenant_header_key: x-scope-orgid + services_query_duration: 72h0m0s + tempo.yaml: | + server: + http_listen_port: 3200 + http_server_read_timeout: 30s + http_server_write_timeout: 30s + internal_server: + enable: true + http_listen_address: 0.0.0.0 + storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/blocks + distributor: + receivers: + otlp: + protocols: + grpc: + tls: + cert_file: /var/run/tls/receiver/grpc/tls.crt + key_file: /var/run/tls/receiver/grpc/tls.key + min_version: "1.3" + http: + tls: + cert_file: /var/run/tls/receiver/http/tls.crt + key_file: /var/run/tls/receiver/http/tls.key + min_version: "1.3" + usage_report: + reporting_enabled: false +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/component: config + app.kubernetes.io/instance: mono + app.kubernetes.io/managed-by: tempo-operator + app.kubernetes.io/name: tempo-monolithic + name: tempo-mono-config + diff --git a/tests/e2e-openshift/tls-singletenant/01-assert.yaml b/tests/e2e-openshift/tls-singletenant/01-assert.yaml index 9b312a0e2..523f74e6a 100644 --- a/tests/e2e-openshift/tls-singletenant/01-assert.yaml +++ b/tests/e2e-openshift/tls-singletenant/01-assert.yaml @@ -87,6 +87,18 @@ spec: name: tempo-simplest-ca-bundle - mountPath: /var/run/tls/server name: tempo-simplest-query-frontend-mtls + - args: + - --cookie-secret-file=/var/run/secrets/kubernetes.io/serviceaccount/token + - --https-address=:8443 + - --openshift-service-account=tempo-simplest-query-frontend + - --provider=openshift + - --tls-cert=/etc/tls/private/tls.crt + - --tls-key=/etc/tls/private/tls.key + - --upstream=http://localhost:16686 + - --upstream-timeout=30s + - '--openshift-sar={"namespace": "chainsaw-tls-singletenant", "resource": + "pods", "verb": "get"}' + name: oauth-proxy volumes: - configMap: defaultMode: 420 @@ -104,15 +116,133 @@ spec: secret: defaultMode: 420 secretName: tempo-simplest-query-frontend-mtls + - name: simplest-ui-oauth-proxy-tls + secret: + defaultMode: 420 + secretName: simplest-ui-oauth-proxy-tls status: availableReplicas: 1 readyReplicas: 1 replicas: 1 + --- -apiVersion: apps/v1 -kind: StatefulSet +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + annotations: + haproxy.router.openshift.io/timeout: 30s + labels: + app.kubernetes.io/component: query-frontend + app.kubernetes.io/instance: simplest + app.kubernetes.io/managed-by: tempo-operator + app.kubernetes.io/name: tempo + name: tempo-simplest-query-frontend +spec: + port: + targetPort: oauth-proxy + tls: + termination: reencrypt + to: + kind: Service + name: tempo-simplest-query-frontend + weight: 100 + wildcardPolicy: None + +--- +apiVersion: v1 +data: + overrides.yaml: | + overrides: + tempo-query-frontend.yaml: "compactor:\n compaction:\n block_retention: 48h0m0s\n + \ ring:\n kvstore:\n store: memberlist\ndistributor:\n receivers:\n jaeger:\n + \ protocols:\n thrift_http:\n endpoint: 0.0.0.0:14268\n tls:\n + \ cert_file: /var/run/tls/receiver/tls.crt\n key_file: /var/run/tls/receiver/tls.key\n + \ min_version: \n thrift_binary:\n endpoint: 0.0.0.0:6832\n + \ thrift_compact:\n endpoint: 0.0.0.0:6831\n grpc:\n endpoint: + 0.0.0.0:14250\n tls:\n cert_file: /var/run/tls/receiver/tls.crt\n + \ key_file: /var/run/tls/receiver/tls.key\n min_version: + \n zipkin:\n tls:\n cert_file: /var/run/tls/receiver/tls.crt\n + \ key_file: /var/run/tls/receiver/tls.key\n min_version: \n otlp:\n + \ protocols:\n grpc:\n endpoint: 0.0.0.0:4317\n tls:\n + \ cert_file: /var/run/tls/receiver/tls.crt\n key_file: /var/run/tls/receiver/tls.key\n + \ min_version: \n http:\n endpoint: 0.0.0.0:4318\n tls:\n + \ cert_file: /var/run/tls/receiver/tls.crt\n key_file: /var/run/tls/receiver/tls.key\n + \ min_version: \n ring:\n kvstore:\n store: memberlist\ningester:\n + \ lifecycler:\n ring:\n kvstore:\n store: memberlist\n replication_factor: + 1\n tokens_file_path: /var/tempo/tokens.json\n max_block_duration: 10m\nmemberlist:\n + \ abort_if_cluster_join_fails: false\n join_members:\n - tempo-simplest-gossip-ring\nmultitenancy_enabled: + false\nquerier:\n max_concurrent_queries: 20\n frontend_worker:\n frontend_address: + tempo-simplest-query-frontend-discovery:9095\n grpc_client_config:\n tls_enabled: + true\n tls_cert_path: /var/run/tls/server/tls.crt\n tls_key_path: /var/run/tls/server/tls.key\n + \ tls_ca_path: /var/run/ca/service-ca.crt\n tls_server_name: tempo-simplest-query-frontend.chainsaw-tls-singletenant.svc.cluster.local\n + \ tls_min_version: VersionTLS13\n search:\n external_hedge_requests_at: + 8s\n external_hedge_requests_up_to: 2\nserver:\n grpc_server_max_recv_msg_size: + 4194304\n grpc_server_max_send_msg_size: 4194304\n http_listen_port: 3200\n + \ http_server_read_timeout: 30s\n http_server_write_timeout: 30s\n log_format: + logfmt\n tls_min_version: VersionTLS13\n grpc_tls_config:\n cert_file: /var/run/tls/server/tls.crt\n + \ key_file: /var/run/tls/server/tls.key\n client_ca_file: /var/run/ca/service-ca.crt\n + \ client_auth_type: RequireAndVerifyClientCert\nstorage:\n trace:\n backend: + s3\n blocklist_poll: 5m\n cache: none\n s3:\n endpoint: minio:9000\n + \ bucket: tempo\n insecure: true\n local:\n path: /var/tempo/traces\n + \ wal:\n path: /var/tempo/wal\nusage_report:\n reporting_enabled: false\nquery_frontend:\n + \ search:\n concurrent_jobs: 2000\n max_duration: 0s\n default_result_limit: + 20\ningester_client:\n grpc_client_config:\n tls_enabled: true\n tls_cert_path: + \ /var/run/tls/server/tls.crt\n tls_key_path: /var/run/tls/server/tls.key\n + \ tls_ca_path: /var/run/ca/service-ca.crt\n tls_server_name: tempo-simplest-ingester.chainsaw-tls-singletenant.svc.cluster.local\n + \ tls_insecure_skip_verify: false\n tls_min_version: VersionTLS13\n" + tempo-query.yaml: | + address: 127.0.0.1:7777 + backend: 127.0.0.1:3200 + tenant_header_key: x-scope-orgid + services_query_duration: 72h0m0s + tempo.yaml: "compactor:\n compaction:\n block_retention: 48h0m0s\n ring:\n + \ kvstore:\n store: memberlist\ndistributor:\n receivers:\n jaeger:\n + \ protocols:\n thrift_http:\n endpoint: 0.0.0.0:14268\n tls:\n + \ cert_file: /var/run/tls/receiver/tls.crt\n key_file: /var/run/tls/receiver/tls.key\n + \ min_version: \n thrift_binary:\n endpoint: 0.0.0.0:6832\n + \ thrift_compact:\n endpoint: 0.0.0.0:6831\n grpc:\n endpoint: + 0.0.0.0:14250\n tls:\n cert_file: /var/run/tls/receiver/tls.crt\n + \ key_file: /var/run/tls/receiver/tls.key\n min_version: + \n zipkin:\n tls:\n cert_file: /var/run/tls/receiver/tls.crt\n + \ key_file: /var/run/tls/receiver/tls.key\n min_version: \n otlp:\n + \ protocols:\n grpc:\n endpoint: 0.0.0.0:4317\n tls:\n + \ cert_file: /var/run/tls/receiver/tls.crt\n key_file: /var/run/tls/receiver/tls.key\n + \ min_version: \n http:\n endpoint: 0.0.0.0:4318\n tls:\n + \ cert_file: /var/run/tls/receiver/tls.crt\n key_file: /var/run/tls/receiver/tls.key\n + \ min_version: \n ring:\n kvstore:\n store: memberlist\ningester:\n + \ lifecycler:\n ring:\n kvstore:\n store: memberlist\n replication_factor: + 1\n tokens_file_path: /var/tempo/tokens.json\n max_block_duration: 10m\nmemberlist:\n + \ abort_if_cluster_join_fails: false\n join_members:\n - tempo-simplest-gossip-ring\nmultitenancy_enabled: + false\nquerier:\n max_concurrent_queries: 20\n frontend_worker:\n frontend_address: + tempo-simplest-query-frontend-discovery:9095\n grpc_client_config:\n tls_enabled: + true\n tls_cert_path: /var/run/tls/server/tls.crt\n tls_key_path: /var/run/tls/server/tls.key\n + \ tls_ca_path: /var/run/ca/service-ca.crt\n tls_server_name: tempo-simplest-query-frontend.chainsaw-tls-singletenant.svc.cluster.local\n + \ tls_min_version: VersionTLS13\n search:\n external_hedge_requests_at: + 8s\n external_hedge_requests_up_to: 2\ninternal_server:\n enable: true\n http_listen_address: + \"\"\n tls_min_version: VersionTLS13\n http_tls_config:\n cert_file: /var/run/tls/server/tls.crt\n + \ key_file: /var/run/tls/server/tls.key\nserver:\n grpc_server_max_recv_msg_size: + 4194304\n grpc_server_max_send_msg_size: 4194304\n http_listen_port: 3200\n + \ http_server_read_timeout: 30s\n http_server_write_timeout: 30s\n log_format: + logfmt\n tls_min_version: VersionTLS13\n grpc_tls_config:\n cert_file: /var/run/tls/server/tls.crt\n + \ key_file: /var/run/tls/server/tls.key\n client_ca_file: /var/run/ca/service-ca.crt\n + \ client_auth_type: RequireAndVerifyClientCert\n http_tls_config:\n cert_file: + \ /var/run/tls/server/tls.crt\n client_auth_type: RequireAndVerifyClientCert\n + \ key_file: /var/run/tls/server/tls.key\n client_ca_file: /var/run/ca/service-ca.crt\nstorage:\n + \ trace:\n backend: s3\n blocklist_poll: 5m\n cache: none\n s3:\n + \ endpoint: minio:9000\n bucket: tempo\n insecure: true\n local:\n + \ path: /var/tempo/traces\n wal:\n path: /var/tempo/wal\nusage_report:\n + \ reporting_enabled: false\nquery_frontend:\n search:\n concurrent_jobs: 2000\n + \ max_duration: 0s\n default_result_limit: 20\ningester_client:\n grpc_client_config:\n + \ tls_enabled: true\n tls_cert_path: /var/run/tls/server/tls.crt\n tls_key_path: + /var/run/tls/server/tls.key\n tls_ca_path: /var/run/ca/service-ca.crt\n tls_server_name: + tempo-simplest-ingester.chainsaw-tls-singletenant.svc.cluster.local\n tls_insecure_skip_verify: + false\n tls_min_version: VersionTLS13\n" +kind: ConfigMap metadata: - name: tempo-simplest-ingester + labels: + app.kubernetes.io/component: config + app.kubernetes.io/instance: simplest + app.kubernetes.io/managed-by: tempo-operator + app.kubernetes.io/name: tempo + name: tempo-simplest namespace: chainsaw-tls-singletenant -status: - readyReplicas: 1 diff --git a/tests/e2e-openshift/tls-singletenant/01-install-tempo.yaml b/tests/e2e-openshift/tls-singletenant/01-install-tempo.yaml index ebb1d61e3..84e2a2571 100644 --- a/tests/e2e-openshift/tls-singletenant/01-install-tempo.yaml +++ b/tests/e2e-openshift/tls-singletenant/01-install-tempo.yaml @@ -1,4 +1,3 @@ -# based on config/samples/openshift/tempo_v1alpha1_multitenancy.yaml apiVersion: tempo.grafana.com/v1alpha1 kind: TempoStack metadata: @@ -22,3 +21,5 @@ spec: queryFrontend: jaegerQuery: enabled: true + ingress: + type: route From 5bfe5d8bdf77267f45693d87788901b47c264022 Mon Sep 17 00:00:00 2001 From: "Ben B." Date: Tue, 8 Oct 2024 13:09:58 +0200 Subject: [PATCH 05/10] jaeger-query: upgrade to version 1.62 (#1050) Signed-off-by: Benedikt Bongartz --- .chloggen/bump_jaeger-to-1.62.yaml | 16 ++++++++++++++++ Makefile | 5 ++--- .../tempo-operator.clusterserviceversion.yaml | 4 ++-- .../tempo-operator.clusterserviceversion.yaml | 4 ++-- config/manager/manager.yaml | 2 +- 5 files changed, 23 insertions(+), 8 deletions(-) create mode 100755 .chloggen/bump_jaeger-to-1.62.yaml diff --git a/.chloggen/bump_jaeger-to-1.62.yaml b/.chloggen/bump_jaeger-to-1.62.yaml new file mode 100755 index 000000000..96aa8c82d --- /dev/null +++ b/.chloggen/bump_jaeger-to-1.62.yaml @@ -0,0 +1,16 @@ +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. tempostack, tempomonolithic, github action) +component: tempostack, tempomonolithic + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: bump jaeger to v1.62 + +# One or more tracking issues related to the change +issues: [1050] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: diff --git a/Makefile b/Makefile index f0a5bd809..8a459b1cd 100644 --- a/Makefile +++ b/Makefile @@ -2,8 +2,7 @@ OPERATOR_VERSION ?= 0.13.0 TEMPO_VERSION ?= 2.5.0 TEMPO_QUERY_VERSION ?= main-2999520 -# TODO change this to a release version. This has https://github.com/jaegertracing/jaeger/commit/d6631f5f2370cfc3a49efce312491031fb387600 -JAEGER_QUERY_VERSION ?= d6631f5f2370cfc3a49efce312491031fb387600 +JAEGER_QUERY_VERSION ?= 1.62.0 TEMPO_GATEWAY_VERSION ?= main-2024-08-05-11d0d94 TEMPO_GATEWAY_OPA_VERSION ?= main-2024-04-29-914c13f OAUTH_PROXY_VERSION=4.14 @@ -12,7 +11,7 @@ MIN_KUBERNETES_VERSION ?= 1.25.0 MIN_OPENSHIFT_VERSION ?= 4.12 TEMPO_IMAGE ?= docker.io/grafana/tempo:$(TEMPO_VERSION) -JAEGER_QUERY_IMAGE ?= docker.io/jaegertracing/jaeger-query-snapshot:$(JAEGER_QUERY_VERSION) +JAEGER_QUERY_IMAGE ?= docker.io/jaegertracing/jaeger-query:$(JAEGER_QUERY_VERSION) TEMPO_QUERY_IMAGE ?= docker.io/grafana/tempo-query:$(TEMPO_QUERY_VERSION) TEMPO_GATEWAY_IMAGE ?= quay.io/observatorium/api:$(TEMPO_GATEWAY_VERSION) TEMPO_GATEWAY_OPA_IMAGE ?= quay.io/observatorium/opa-openshift:$(TEMPO_GATEWAY_OPA_VERSION) diff --git a/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml b/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml index 85bb37bf8..aa33b1897 100644 --- a/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml +++ b/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml @@ -1425,7 +1425,7 @@ spec: - name: RELATED_IMAGE_TEMPO value: docker.io/grafana/tempo:2.5.0 - name: RELATED_IMAGE_JAEGER_QUERY - value: docker.io/jaegertracing/jaeger-query-snapshot:d6631f5f2370cfc3a49efce312491031fb387600 + value: docker.io/jaegertracing/jaeger-query:1.62.0 - name: RELATED_IMAGE_TEMPO_QUERY value: docker.io/grafana/tempo-query:main-2999520 - name: RELATED_IMAGE_TEMPO_GATEWAY @@ -1574,7 +1574,7 @@ spec: relatedImages: - image: docker.io/grafana/tempo:2.5.0 name: tempo - - image: docker.io/jaegertracing/jaeger-query-snapshot:d6631f5f2370cfc3a49efce312491031fb387600 + - image: docker.io/jaegertracing/jaeger-query:1.62.0 name: jaeger-query - image: docker.io/grafana/tempo-query:main-2999520 name: tempo-query diff --git a/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml b/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml index a4868857e..5f8dba924 100644 --- a/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml +++ b/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml @@ -1435,7 +1435,7 @@ spec: - name: RELATED_IMAGE_TEMPO value: docker.io/grafana/tempo:2.5.0 - name: RELATED_IMAGE_JAEGER_QUERY - value: docker.io/jaegertracing/jaeger-query-snapshot:d6631f5f2370cfc3a49efce312491031fb387600 + value: docker.io/jaegertracing/jaeger-query:1.62.0 - name: RELATED_IMAGE_TEMPO_QUERY value: docker.io/grafana/tempo-query:main-2999520 - name: RELATED_IMAGE_TEMPO_GATEWAY @@ -1595,7 +1595,7 @@ spec: relatedImages: - image: docker.io/grafana/tempo:2.5.0 name: tempo - - image: docker.io/jaegertracing/jaeger-query-snapshot:d6631f5f2370cfc3a49efce312491031fb387600 + - image: docker.io/jaegertracing/jaeger-query:1.62.0 name: jaeger-query - image: docker.io/grafana/tempo-query:main-2999520 name: tempo-query diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index accf9ad74..4db4cd802 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -42,7 +42,7 @@ spec: - name: RELATED_IMAGE_TEMPO value: docker.io/grafana/tempo:2.5.0 - name: RELATED_IMAGE_JAEGER_QUERY - value: docker.io/jaegertracing/jaeger-query-snapshot:d6631f5f2370cfc3a49efce312491031fb387600 + value: docker.io/jaegertracing/jaeger-query:1.62.0 - name: RELATED_IMAGE_TEMPO_QUERY value: docker.io/grafana/tempo-query:main-2999520 - name: RELATED_IMAGE_TEMPO_GATEWAY From 642fc96b6bce36e7c255c0d83fdfe6b3d1d34e3c Mon Sep 17 00:00:00 2001 From: "Ben B." Date: Tue, 8 Oct 2024 13:18:48 +0200 Subject: [PATCH 06/10] queryfrontend: grant jaeger-query access to pki certs (#1051) * queryfrontend: grant jaeer-query access to pki certs Signed-off-by: Benedikt Bongartz * Update .chloggen/fix_jaeger-query-certs.yaml --------- Signed-off-by: Benedikt Bongartz Co-authored-by: Andreas Gerstmayr --- .chloggen/fix_jaeger-query-certs.yaml | 16 ++++++++++++++++ .../manifests/queryfrontend/query_frontend.go | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100755 .chloggen/fix_jaeger-query-certs.yaml diff --git a/.chloggen/fix_jaeger-query-certs.yaml b/.chloggen/fix_jaeger-query-certs.yaml new file mode 100755 index 000000000..d1ecc3405 --- /dev/null +++ b/.chloggen/fix_jaeger-query-certs.yaml @@ -0,0 +1,16 @@ +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: bug_fix + +# The name of the component, or a single word describing the area of concern, (e.g. tempostack, tempomonolithic, github action) +component: tempostack + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: grant jaeer-query access to pki certs + +# One or more tracking issues related to the change +issues: [1051] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: diff --git a/internal/manifests/queryfrontend/query_frontend.go b/internal/manifests/queryfrontend/query_frontend.go index bb232db0e..27ca29859 100644 --- a/internal/manifests/queryfrontend/query_frontend.go +++ b/internal/manifests/queryfrontend/query_frontend.go @@ -56,7 +56,7 @@ func BuildQueryFrontend(params manifestutils.Params) ([]client.Object, error) { if gates.HTTPEncryption || gates.GRPCEncryption { caBundleName := naming.SigningCABundleName(tempo.Name) - targets := []string{containerNameTempo, containerNameTempoQuery} + targets := []string{containerNameTempo, containerNameJaegerQuery, containerNameTempoQuery} if err := manifestutils.ConfigureServiceCAByContainerName(&d.Spec.Template.Spec, caBundleName, targets...); err != nil { return nil, err } From 88d46a582b84778d6bb410d70f131c8a6493bb00 Mon Sep 17 00:00:00 2001 From: Ishwar Kanse Date: Tue, 8 Oct 2024 20:02:56 +0530 Subject: [PATCH 07/10] Test mut-gather script (#1052) --- .../monolithic-route/chainsaw-test.yaml | 7 ++- .../monolithic-route/check-must-gahter.sh | 44 +++++++++++++++ tests/e2e-openshift/route/chainsaw-test.yaml | 5 ++ .../e2e-openshift/route/check-must-gahter.sh | 53 +++++++++++++++++++ 4 files changed, 108 insertions(+), 1 deletion(-) create mode 100755 tests/e2e-openshift/monolithic-route/check-must-gahter.sh create mode 100755 tests/e2e-openshift/route/check-must-gahter.sh diff --git a/tests/e2e-openshift/monolithic-route/chainsaw-test.yaml b/tests/e2e-openshift/monolithic-route/chainsaw-test.yaml index 2dac7e15c..410f1d13e 100755 --- a/tests/e2e-openshift/monolithic-route/chainsaw-test.yaml +++ b/tests/e2e-openshift/monolithic-route/chainsaw-test.yaml @@ -10,4 +10,9 @@ spec: - apply: file: install-tempo.yaml - assert: - file: install-tempo-assert.yaml \ No newline at end of file + file: install-tempo-assert.yaml + - name: Run the must-gather and verify the contents + try: + - script: + timeout: 5m + content: ./check-must-gahter.sh diff --git a/tests/e2e-openshift/monolithic-route/check-must-gahter.sh b/tests/e2e-openshift/monolithic-route/check-must-gahter.sh new file mode 100755 index 000000000..1cba22ef1 --- /dev/null +++ b/tests/e2e-openshift/monolithic-route/check-must-gahter.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# Check if must gather directory exists +MUST_GATHER_DIR=/tmp/monolithic-route +mkdir -p $MUST_GATHER_DIR + +# Run the must-gather script +oc adm must-gather --dest-dir=$MUST_GATHER_DIR --image=quay.io/rhn_support_ikanse/tempo-must-gather:latest -- /usr/bin/must-gather --operator-namespace tempo-operator + +# Define required files and directories +REQUIRED_ITEMS=( + "event-filter.html" + "timestamp" + "*sha*/deployment-tempo-operator-controller.yaml" + "*sha*/olm/operator-servicemeshoperator-openshift-operators.yaml" + "*sha*/olm/installplan-install-*.yaml" + "*sha*/olm/clusterserviceversion-tempo-operator-*.yaml" + "*sha*/olm/operator-opentelemetry-product-openshift-opentelemetry-operator.yaml" + "*sha*/olm/operator-tempo-operator-tempo-operator.yaml" + "*sha*/olm/operator-tempo-product-openshift-tempo-operator.yaml" + "*sha*/olm/subscription-tempo-operator-*-sub.yaml" + "*sha*/namespaces/chainsaw-mono-route/tempomonolithic/mono-route/tempomonolithic-mono-route.yaml" + "*sha*/namespaces/chainsaw-mono-route/tempomonolithic/mono-route/service-tempo-mono-route-jaegerui.yaml" + "*sha*/namespaces/chainsaw-mono-route/tempomonolithic/mono-route/configmap-tempo-mono-route-serving-cabundle.yaml" + "*sha*/namespaces/chainsaw-mono-route/tempomonolithic/mono-route/statefulset-tempo-mono-route.yaml" + "*sha*/namespaces/chainsaw-mono-route/tempomonolithic/mono-route/service-tempo-mono-route.yaml" + "*sha*/namespaces/chainsaw-mono-route/tempomonolithic/mono-route/route-tempo-mono-route-jaegerui.yaml" + "*sha*/namespaces/chainsaw-mono-route/tempomonolithic/mono-route/configmap-tempo-mono-route-config.yaml" + "*sha*/namespaces/chainsaw-mono-route/tempomonolithic/mono-route/serviceaccount-tempo-mono-route.yaml" + "*sha*/tempo-operator-controller-*" +) + +# Verify each required item +for item in "${REQUIRED_ITEMS[@]}"; do + if ! find "$MUST_GATHER_DIR" -path "$MUST_GATHER_DIR/$item" -print -quit | grep -q .; then + echo "Missing: $item" + exit 1 + else + echo "Found: $item" + fi +done + +# Cleanup the must-gather directory +rm -rf $MUST_GATHER_DIR \ No newline at end of file diff --git a/tests/e2e-openshift/route/chainsaw-test.yaml b/tests/e2e-openshift/route/chainsaw-test.yaml index a3ab3f7e7..c6e031a3f 100755 --- a/tests/e2e-openshift/route/chainsaw-test.yaml +++ b/tests/e2e-openshift/route/chainsaw-test.yaml @@ -23,3 +23,8 @@ spec: - script: timeout: 5m content: kubectl get --namespace $NAMESPACE tempo simplest -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' | grep True + - name: Run the must-gather and verify the contents + try: + - script: + timeout: 5m + content: ./check-must-gahter.sh diff --git a/tests/e2e-openshift/route/check-must-gahter.sh b/tests/e2e-openshift/route/check-must-gahter.sh new file mode 100755 index 000000000..736dff03f --- /dev/null +++ b/tests/e2e-openshift/route/check-must-gahter.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +# Check if must gather directory exists +MUST_GATHER_DIR=/tmp/route +mkdir -p $MUST_GATHER_DIR + +# Run the must-gather script +oc adm must-gather --dest-dir=$MUST_GATHER_DIR --image=quay.io/rhn_support_ikanse/tempo-must-gather:latest -- /usr/bin/must-gather --operator-namespace tempo-operator + +# Define required files and directories +REQUIRED_ITEMS=( + "event-filter.html" + "timestamp" + "*sha*/deployment-tempo-operator-controller.yaml" + "*sha*/olm/installplan-install-*" + "*sha*/olm/clusterserviceversion-tempo-operator-*.yaml" + "*sha*/olm/operator-opentelemetry-product-openshift-opentelemetry-operator.yaml" + "*sha*/olm/operator-tempo-operator-tempo-operator.yaml" + "*sha*/olm/operator-tempo-product-openshift-tempo-operator.yaml" + "*sha*/olm/subscription-tempo-operator-*-sub.yaml" + "*sha*/namespaces/chainsaw-route/tempostack/simplest/service-tempo-simplest-distributor.yaml" + "*sha*/namespaces/chainsaw-route/tempostack/simplest/service-tempo-simplest-ingester.yaml" + "*sha*/namespaces/chainsaw-route/tempostack/simplest/deployment-tempo-simplest-distributor.yaml" + "*sha*/namespaces/chainsaw-route/tempostack/simplest/service-tempo-simplest-querier.yaml" + "*sha*/namespaces/chainsaw-route/tempostack/simplest/configmap-tempo-simplest.yaml" + "*sha*/namespaces/chainsaw-route/tempostack/simplest/service-tempo-simplest-compactor.yaml" + "*sha*/namespaces/chainsaw-route/tempostack/simplest/service-tempo-simplest-query-frontend.yaml" + "*sha*/namespaces/chainsaw-route/tempostack/simplest/deployment-tempo-simplest-querier.yaml" + "*sha*/namespaces/chainsaw-route/tempostack/simplest/tempostack-simplest.yaml" + "*sha*/namespaces/chainsaw-route/tempostack/simplest/serviceaccount-tempo-simplest-query-frontend.yaml" + "*sha*/namespaces/chainsaw-route/tempostack/simplest/statefulset-tempo-simplest-ingester.yaml" + "*sha*/namespaces/chainsaw-route/tempostack/simplest/deployment-tempo-simplest-query-frontend.yaml" + "*sha*/namespaces/chainsaw-route/tempostack/simplest/route-tempo-simplest-query-frontend.yaml" + "*sha*/namespaces/chainsaw-route/tempostack/simplest/service-tempo-simplest-gossip-ring.yaml" + "*sha*/namespaces/chainsaw-route/tempostack/simplest/configmap-tempo-simplest-ca-bundle.yaml" + "*sha*/namespaces/chainsaw-route/tempostack/simplest/serviceaccount-tempo-simplest.yaml" + "*sha*/namespaces/chainsaw-route/tempostack/simplest/deployment-tempo-simplest-compactor.yaml" + "*sha*/namespaces/chainsaw-route/tempostack/simplest/service-tempo-simplest-query-frontend-discovery.yaml" + "*sha*/tempo-operator-controller-*" +) + +# Verify each required item +for item in "${REQUIRED_ITEMS[@]}"; do + if ! find "$MUST_GATHER_DIR" -path "$MUST_GATHER_DIR/$item" -print -quit | grep -q .; then + echo "Missing: $item" + exit 1 + else + echo "Found: $item" + fi +done + +# Cleanup the must-gather directory +rm -rf $MUST_GATHER_DIR \ No newline at end of file From 900170bf85d04a0351c76a249abfb0950bdda4d5 Mon Sep 17 00:00:00 2001 From: Pavol Loffay Date: Thu, 10 Oct 2024 18:26:59 +0200 Subject: [PATCH 08/10] Tempo query improve search performance (#1048) * Tempo query improve search performance Signed-off-by: Pavol Loffay * Fix Signed-off-by: Pavol Loffay --------- Signed-off-by: Pavol Loffay --- .chloggen/tempo-query-find-traces-jobs.yaml | 21 ++++++++++++ Makefile | 2 +- .../v1alpha1/tempomonolithic_defaults.go | 12 ++++--- .../v1alpha1/tempomonolithic_defaults_test.go | 20 ++++++++---- apis/tempo/v1alpha1/tempomonolithic_types.go | 13 ++++++++ apis/tempo/v1alpha1/tempostack_types.go | 13 ++++++++ .../tempo-operator.clusterserviceversion.yaml | 32 +++++++++++++++++-- .../tempo.grafana.com_tempomonolithics.yaml | 11 +++++++ .../tempo.grafana.com_tempostacks.yaml | 11 +++++++ .../tempo-operator.clusterserviceversion.yaml | 32 +++++++++++++++++-- .../tempo.grafana.com_tempomonolithics.yaml | 11 +++++++ .../tempo.grafana.com_tempostacks.yaml | 11 +++++++ .../tempo.grafana.com_tempomonolithics.yaml | 11 +++++++ .../bases/tempo.grafana.com_tempostacks.yaml | 11 +++++++ config/manager/manager.yaml | 2 +- .../tempo-operator.clusterserviceversion.yaml | 26 +++++++++++++++ .../tempo-operator.clusterserviceversion.yaml | 26 +++++++++++++++ .../tempo.grafana.com_tempomonolithics.yaml | 1 + docs/spec/tempo.grafana.com_tempostacks.yaml | 1 + internal/manifests/config/build.go | 15 +++++++-- internal/manifests/config/options.go | 13 ++++---- internal/manifests/config/tempo-query.yaml | 1 + internal/manifests/monolithic/configmap.go | 10 +++--- .../manifests/monolithic/configmap_test.go | 6 ++-- .../install-tempo-assert.yaml | 1 + .../install-tempostack-assert.yaml | 1 + 26 files changed, 280 insertions(+), 34 deletions(-) create mode 100755 .chloggen/tempo-query-find-traces-jobs.yaml diff --git a/.chloggen/tempo-query-find-traces-jobs.yaml b/.chloggen/tempo-query-find-traces-jobs.yaml new file mode 100755 index 000000000..294deef9f --- /dev/null +++ b/.chloggen/tempo-query-find-traces-jobs.yaml @@ -0,0 +1,21 @@ +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. tempostack, tempomonolithic, github action) +component: tempostack, tempomonolithic + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Add tempo-query CRD option to speed up trace search. + +# One or more tracking issues related to the change +issues: [1048] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: | + Following CRD options were added to speed up trace search in Jaeger UI/API. The trace search first + searches for traceids and then it gets a full trace. With this configuration option the requests + to get the full trace can be run in parallel: + For `TempoStack` - `spec.template.queryFrontend.jaegerQuery.findTracesConcurrentRequests` + For `TempoMonolithic` - `spec.jaegerui.findTracesConcurrentRequests` diff --git a/Makefile b/Makefile index 8a459b1cd..8b5c89e13 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,8 @@ # Current Operator version OPERATOR_VERSION ?= 0.13.0 TEMPO_VERSION ?= 2.5.0 -TEMPO_QUERY_VERSION ?= main-2999520 JAEGER_QUERY_VERSION ?= 1.62.0 +TEMPO_QUERY_VERSION ?= main-1de25ca TEMPO_GATEWAY_VERSION ?= main-2024-08-05-11d0d94 TEMPO_GATEWAY_OPA_VERSION ?= main-2024-04-29-914c13f OAUTH_PROXY_VERSION=4.14 diff --git a/apis/tempo/v1alpha1/tempomonolithic_defaults.go b/apis/tempo/v1alpha1/tempomonolithic_defaults.go index 39fc395f9..148422d60 100644 --- a/apis/tempo/v1alpha1/tempomonolithic_defaults.go +++ b/apis/tempo/v1alpha1/tempomonolithic_defaults.go @@ -13,10 +13,11 @@ import ( ) var ( - twoGBQuantity = resource.MustParse("2Gi") - tenGBQuantity = resource.MustParse("10Gi") - defaultServicesDuration = metav1.Duration{Duration: time.Hour * 24 * 3} - defaultTimeout = metav1.Duration{Duration: time.Second * 30} + twoGBQuantity = resource.MustParse("2Gi") + tenGBQuantity = resource.MustParse("10Gi") + defaultServicesDuration = metav1.Duration{Duration: time.Hour * 24 * 3} + defaultTimeout = metav1.Duration{Duration: time.Second * 30} + defaultFindTracesConcurrentRequests = 2 ) // Default sets all default values in a central place, instead of setting it at every place where the value is accessed. @@ -88,6 +89,9 @@ func (r *TempoMonolithic) Default(ctrlConfig configv1alpha1.ProjectConfig) { if r.Spec.JaegerUI.ServicesQueryDuration == nil { r.Spec.JaegerUI.ServicesQueryDuration = &defaultServicesDuration } + if r.Spec.JaegerUI.FindTracesConcurrentRequests == 0 { + r.Spec.JaegerUI.FindTracesConcurrentRequests = defaultFindTracesConcurrentRequests + } } if r.Spec.Timeout.Duration == 0 { diff --git a/apis/tempo/v1alpha1/tempomonolithic_defaults_test.go b/apis/tempo/v1alpha1/tempomonolithic_defaults_test.go index be28ff060..0768c3688 100644 --- a/apis/tempo/v1alpha1/tempomonolithic_defaults_test.go +++ b/apis/tempo/v1alpha1/tempomonolithic_defaults_test.go @@ -196,7 +196,8 @@ func TestMonolithicDefault(t *testing.T) { Enabled: true, SAR: "{\"namespace\": \"testns\", \"resource\": \"pods\", \"verb\": \"get\"}", }, - ServicesQueryDuration: &defaultServicesDuration, + ServicesQueryDuration: &defaultServicesDuration, + FindTracesConcurrentRequests: 2, }, Management: "Managed", Timeout: metav1.Duration{Duration: time.Second * 30}, @@ -269,7 +270,8 @@ func TestMonolithicDefault(t *testing.T) { Enabled: false, SAR: "{\"namespace\": \"testns\", \"resource\": \"pods\", \"verb\": \"get\"}", }, - ServicesQueryDuration: &defaultServicesDuration, + ServicesQueryDuration: &defaultServicesDuration, + FindTracesConcurrentRequests: 2, }, Management: "Managed", Timeout: metav1.Duration{Duration: time.Second * 30}, @@ -334,7 +336,8 @@ func TestMonolithicDefault(t *testing.T) { Enabled: true, SAR: "{\"namespace\": \"testns\", \"resource\": \"pods\", \"verb\": \"get\"}", }, - ServicesQueryDuration: &defaultServicesDuration, + ServicesQueryDuration: &defaultServicesDuration, + FindTracesConcurrentRequests: 2, }, Management: "Managed", Timeout: metav1.Duration{Duration: time.Second * 30}, @@ -398,7 +401,8 @@ func TestMonolithicDefault(t *testing.T) { Enabled: false, SAR: "{\"namespace\": \"testns\", \"resource\": \"pods\", \"verb\": \"get\"}", }, - ServicesQueryDuration: &defaultServicesDuration, + ServicesQueryDuration: &defaultServicesDuration, + FindTracesConcurrentRequests: 2, }, Management: "Managed", Timeout: metav1.Duration{Duration: time.Second * 30}, @@ -406,7 +410,7 @@ func TestMonolithicDefault(t *testing.T) { }, }, { - name: "define custom duration for services list and timeout", + name: "define custom duration for services list, timeout and find traces", input: &TempoMonolithic{ ObjectMeta: v1.ObjectMeta{ Name: "test", @@ -424,7 +428,8 @@ func TestMonolithicDefault(t *testing.T) { Route: &MonolithicJaegerUIRouteSpec{ Enabled: true, }, - ServicesQueryDuration: &v1.Duration{Duration: time.Duration(100 * 100)}, + ServicesQueryDuration: &v1.Duration{Duration: time.Duration(100 * 100)}, + FindTracesConcurrentRequests: 40, }, Timeout: metav1.Duration{Duration: time.Hour}, }, @@ -461,7 +466,8 @@ func TestMonolithicDefault(t *testing.T) { Enabled: false, SAR: "{\"namespace\": \"testns\", \"resource\": \"pods\", \"verb\": \"get\"}", }, - ServicesQueryDuration: &v1.Duration{Duration: time.Duration(100 * 100)}, + ServicesQueryDuration: &v1.Duration{Duration: time.Duration(100 * 100)}, + FindTracesConcurrentRequests: 40, }, Management: "Managed", Timeout: metav1.Duration{Duration: time.Hour}, diff --git a/apis/tempo/v1alpha1/tempomonolithic_types.go b/apis/tempo/v1alpha1/tempomonolithic_types.go index fef946876..b6db8fe0a 100644 --- a/apis/tempo/v1alpha1/tempomonolithic_types.go +++ b/apis/tempo/v1alpha1/tempomonolithic_types.go @@ -261,6 +261,19 @@ type MonolithicJaegerUISpec struct { // +optional // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="ServicesQueryDuration",xDescriptors="urn:alm:descriptor:com.tectonic.ui:advanced" ServicesQueryDuration *metav1.Duration `json:"servicesQueryDuration,omitempty"` + + // FindTracesConcurrentRequests defines how many concurrent request a single trace search can submit (defaults 2). + // The search for traces in Jaeger submits limit+1 requests. First requests finds trace IDs and then it fetches + // entire traces by ID. This property allows Jaeger to fetch traces in parallel. + // Note that by default a single Tempo querier can process 20 concurrent search jobs. + // Increasing this property might require scaling up querier instances, especially on error "job queue full" + // See also Tempo's extraConfig: + // querier.max_concurrent_queries (20 default) + // query_frontend.max_outstanding_per_tenant: (2000 default). Increase if the query-frontend returns 429 + // + // +optional + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="FindTracesConcurrentRequests",xDescriptors="urn:alm:descriptor:com.tectonic.ui:advanced" + FindTracesConcurrentRequests int `json:"findTracesConcurrentRequests,omitempty"` } // MonolithicJaegerUIIngressSpec defines the settings for the Jaeger UI ingress. diff --git a/apis/tempo/v1alpha1/tempostack_types.go b/apis/tempo/v1alpha1/tempostack_types.go index cc0186ab0..e868b207d 100644 --- a/apis/tempo/v1alpha1/tempostack_types.go +++ b/apis/tempo/v1alpha1/tempostack_types.go @@ -618,6 +618,19 @@ type JaegerQuerySpec struct { // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="ServicesQueryDuration" ServicesQueryDuration *metav1.Duration `json:"servicesQueryDuration,omitempty"` + // FindTracesConcurrentRequests defines how many concurrent request a single trace search can submit (defaults querier.replicas*2). + // The search for traces in Jaeger submits limit+1 requests. First requests finds trace IDs and then it fetches + // entire traces by ID. This property allows Jaeger to fetch traces in parallel. + // Note that by default a single Tempo querier can process 20 concurrent search jobs. + // Increasing this property might require scaling up querier instances, especially on error "job queue full" + // See also Tempo's extraConfig: + // querier.max_concurrent_queries (20 default) + // query_frontend.max_outstanding_per_tenant: (2000 default). Increase if the query-frontend returns 429 + // + // +optional + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="FindTracesConcurrentRequests",xDescriptors="urn:alm:descriptor:com.tectonic.ui:advanced" + FindTracesConcurrentRequests int `json:"findTracesConcurrentRequests,omitempty"` + // Authentication defines the options for the oauth proxy used to protect jaeger UI // // +optional diff --git a/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml b/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml index aa33b1897..e597be8a4 100644 --- a/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml +++ b/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml @@ -74,7 +74,7 @@ metadata: capabilities: Deep Insights categories: Logging & Tracing,Monitoring containerImage: ghcr.io/grafana/tempo-operator/tempo-operator:v0.13.0 - createdAt: "2024-10-07T07:11:28Z" + createdAt: "2024-10-10T15:57:46Z" description: Create and manage deployments of Tempo, a high-scale distributed tracing backend. operatorframework.io/cluster-monitoring: "true" @@ -307,6 +307,19 @@ spec: "{"namespace": "", "resource": "pods", "verb": "get"}' displayName: SAR path: jaegerui.authentication.sar + - description: 'FindTracesConcurrentRequests defines how many concurrent request + a single trace search can submit (defaults 2). The search for traces in + Jaeger submits limit+1 requests. First requests finds trace IDs and then + it fetches entire traces by ID. This property allows Jaeger to fetch traces + in parallel. Note that by default a single Tempo querier can process 20 + concurrent search jobs. Increasing this property might require scaling up + querier instances, especially on error "job queue full" See also Tempo''s + extraConfig: querier.max_concurrent_queries (20 default) query_frontend.max_outstanding_per_tenant: + (2000 default). Increase if the query-frontend returns 429' + displayName: FindTracesConcurrentRequests + path: jaegerui.findTracesConcurrentRequests + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced - description: Annotations defines the annotations of the Ingress object. displayName: Annotations path: jaegerui.ingress.annotations @@ -1002,6 +1015,19 @@ spec: path: template.queryFrontend.jaegerQuery.enabled x-descriptors: - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - description: 'FindTracesConcurrentRequests defines how many concurrent request + a single trace search can submit (defaults querier.replicas*2). The search + for traces in Jaeger submits limit+1 requests. First requests finds trace + IDs and then it fetches entire traces by ID. This property allows Jaeger + to fetch traces in parallel. Note that by default a single Tempo querier + can process 20 concurrent search jobs. Increasing this property might require + scaling up querier instances, especially on error "job queue full" See also + Tempo''s extraConfig: querier.max_concurrent_queries (20 default) query_frontend.max_outstanding_per_tenant: + (2000 default). Increase if the query-frontend returns 429' + displayName: FindTracesConcurrentRequests + path: template.queryFrontend.jaegerQuery.findTracesConcurrentRequests + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced - description: Ingress defines the options for the Jaeger Query ingress. displayName: Jaeger Query UI Ingress Settings path: template.queryFrontend.jaegerQuery.ingress @@ -1427,7 +1453,7 @@ spec: - name: RELATED_IMAGE_JAEGER_QUERY value: docker.io/jaegertracing/jaeger-query:1.62.0 - name: RELATED_IMAGE_TEMPO_QUERY - value: docker.io/grafana/tempo-query:main-2999520 + value: docker.io/grafana/tempo-query:main-1de25ca - name: RELATED_IMAGE_TEMPO_GATEWAY value: quay.io/observatorium/api:main-2024-08-05-11d0d94 - name: RELATED_IMAGE_TEMPO_GATEWAY_OPA @@ -1576,7 +1602,7 @@ spec: name: tempo - image: docker.io/jaegertracing/jaeger-query:1.62.0 name: jaeger-query - - image: docker.io/grafana/tempo-query:main-2999520 + - image: docker.io/grafana/tempo-query:main-1de25ca name: tempo-query - image: quay.io/observatorium/api:main-2024-08-05-11d0d94 name: tempo-gateway diff --git a/bundle/community/manifests/tempo.grafana.com_tempomonolithics.yaml b/bundle/community/manifests/tempo.grafana.com_tempomonolithics.yaml index 115daff46..6d285729c 100644 --- a/bundle/community/manifests/tempo.grafana.com_tempomonolithics.yaml +++ b/bundle/community/manifests/tempo.grafana.com_tempomonolithics.yaml @@ -1109,6 +1109,17 @@ spec: description: Enabled defines if the Jaeger UI component should be created. type: boolean + findTracesConcurrentRequests: + description: |- + FindTracesConcurrentRequests defines how many concurrent request a single trace search can submit (defaults 2). + The search for traces in Jaeger submits limit+1 requests. First requests finds trace IDs and then it fetches + entire traces by ID. This property allows Jaeger to fetch traces in parallel. + Note that by default a single Tempo querier can process 20 concurrent search jobs. + Increasing this property might require scaling up querier instances, especially on error "job queue full" + See also Tempo's extraConfig: + querier.max_concurrent_queries (20 default) + query_frontend.max_outstanding_per_tenant: (2000 default). Increase if the query-frontend returns 429 + type: integer ingress: description: Ingress defines the Ingress configuration for the Jaeger UI. diff --git a/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml b/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml index eaede14e7..a34011a10 100644 --- a/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml +++ b/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml @@ -2391,6 +2391,17 @@ spec: description: Enabled defines if the Jaeger Query component should be created. type: boolean + findTracesConcurrentRequests: + description: |- + FindTracesConcurrentRequests defines how many concurrent request a single trace search can submit (defaults querier.replicas*2). + The search for traces in Jaeger submits limit+1 requests. First requests finds trace IDs and then it fetches + entire traces by ID. This property allows Jaeger to fetch traces in parallel. + Note that by default a single Tempo querier can process 20 concurrent search jobs. + Increasing this property might require scaling up querier instances, especially on error "job queue full" + See also Tempo's extraConfig: + querier.max_concurrent_queries (20 default) + query_frontend.max_outstanding_per_tenant: (2000 default). Increase if the query-frontend returns 429 + type: integer ingress: description: Ingress defines the options for the Jaeger Query ingress. diff --git a/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml b/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml index 5f8dba924..172fe9a94 100644 --- a/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml +++ b/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml @@ -74,7 +74,7 @@ metadata: capabilities: Deep Insights categories: Logging & Tracing,Monitoring containerImage: ghcr.io/grafana/tempo-operator/tempo-operator:v0.13.0 - createdAt: "2024-10-07T07:11:27Z" + createdAt: "2024-10-10T15:57:44Z" description: Create and manage deployments of Tempo, a high-scale distributed tracing backend. operatorframework.io/cluster-monitoring: "true" @@ -307,6 +307,19 @@ spec: "{"namespace": "", "resource": "pods", "verb": "get"}' displayName: SAR path: jaegerui.authentication.sar + - description: 'FindTracesConcurrentRequests defines how many concurrent request + a single trace search can submit (defaults 2). The search for traces in + Jaeger submits limit+1 requests. First requests finds trace IDs and then + it fetches entire traces by ID. This property allows Jaeger to fetch traces + in parallel. Note that by default a single Tempo querier can process 20 + concurrent search jobs. Increasing this property might require scaling up + querier instances, especially on error "job queue full" See also Tempo''s + extraConfig: querier.max_concurrent_queries (20 default) query_frontend.max_outstanding_per_tenant: + (2000 default). Increase if the query-frontend returns 429' + displayName: FindTracesConcurrentRequests + path: jaegerui.findTracesConcurrentRequests + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced - description: Annotations defines the annotations of the Ingress object. displayName: Annotations path: jaegerui.ingress.annotations @@ -1002,6 +1015,19 @@ spec: path: template.queryFrontend.jaegerQuery.enabled x-descriptors: - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - description: 'FindTracesConcurrentRequests defines how many concurrent request + a single trace search can submit (defaults querier.replicas*2). The search + for traces in Jaeger submits limit+1 requests. First requests finds trace + IDs and then it fetches entire traces by ID. This property allows Jaeger + to fetch traces in parallel. Note that by default a single Tempo querier + can process 20 concurrent search jobs. Increasing this property might require + scaling up querier instances, especially on error "job queue full" See also + Tempo''s extraConfig: querier.max_concurrent_queries (20 default) query_frontend.max_outstanding_per_tenant: + (2000 default). Increase if the query-frontend returns 429' + displayName: FindTracesConcurrentRequests + path: template.queryFrontend.jaegerQuery.findTracesConcurrentRequests + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced - description: Ingress defines the options for the Jaeger Query ingress. displayName: Jaeger Query UI Ingress Settings path: template.queryFrontend.jaegerQuery.ingress @@ -1437,7 +1463,7 @@ spec: - name: RELATED_IMAGE_JAEGER_QUERY value: docker.io/jaegertracing/jaeger-query:1.62.0 - name: RELATED_IMAGE_TEMPO_QUERY - value: docker.io/grafana/tempo-query:main-2999520 + value: docker.io/grafana/tempo-query:main-1de25ca - name: RELATED_IMAGE_TEMPO_GATEWAY value: quay.io/observatorium/api:main-2024-08-05-11d0d94 - name: RELATED_IMAGE_TEMPO_GATEWAY_OPA @@ -1597,7 +1623,7 @@ spec: name: tempo - image: docker.io/jaegertracing/jaeger-query:1.62.0 name: jaeger-query - - image: docker.io/grafana/tempo-query:main-2999520 + - image: docker.io/grafana/tempo-query:main-1de25ca name: tempo-query - image: quay.io/observatorium/api:main-2024-08-05-11d0d94 name: tempo-gateway diff --git a/bundle/openshift/manifests/tempo.grafana.com_tempomonolithics.yaml b/bundle/openshift/manifests/tempo.grafana.com_tempomonolithics.yaml index 115daff46..6d285729c 100644 --- a/bundle/openshift/manifests/tempo.grafana.com_tempomonolithics.yaml +++ b/bundle/openshift/manifests/tempo.grafana.com_tempomonolithics.yaml @@ -1109,6 +1109,17 @@ spec: description: Enabled defines if the Jaeger UI component should be created. type: boolean + findTracesConcurrentRequests: + description: |- + FindTracesConcurrentRequests defines how many concurrent request a single trace search can submit (defaults 2). + The search for traces in Jaeger submits limit+1 requests. First requests finds trace IDs and then it fetches + entire traces by ID. This property allows Jaeger to fetch traces in parallel. + Note that by default a single Tempo querier can process 20 concurrent search jobs. + Increasing this property might require scaling up querier instances, especially on error "job queue full" + See also Tempo's extraConfig: + querier.max_concurrent_queries (20 default) + query_frontend.max_outstanding_per_tenant: (2000 default). Increase if the query-frontend returns 429 + type: integer ingress: description: Ingress defines the Ingress configuration for the Jaeger UI. diff --git a/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml b/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml index eaede14e7..a34011a10 100644 --- a/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml +++ b/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml @@ -2391,6 +2391,17 @@ spec: description: Enabled defines if the Jaeger Query component should be created. type: boolean + findTracesConcurrentRequests: + description: |- + FindTracesConcurrentRequests defines how many concurrent request a single trace search can submit (defaults querier.replicas*2). + The search for traces in Jaeger submits limit+1 requests. First requests finds trace IDs and then it fetches + entire traces by ID. This property allows Jaeger to fetch traces in parallel. + Note that by default a single Tempo querier can process 20 concurrent search jobs. + Increasing this property might require scaling up querier instances, especially on error "job queue full" + See also Tempo's extraConfig: + querier.max_concurrent_queries (20 default) + query_frontend.max_outstanding_per_tenant: (2000 default). Increase if the query-frontend returns 429 + type: integer ingress: description: Ingress defines the options for the Jaeger Query ingress. diff --git a/config/crd/bases/tempo.grafana.com_tempomonolithics.yaml b/config/crd/bases/tempo.grafana.com_tempomonolithics.yaml index 92a7a6753..918f93e42 100644 --- a/config/crd/bases/tempo.grafana.com_tempomonolithics.yaml +++ b/config/crd/bases/tempo.grafana.com_tempomonolithics.yaml @@ -1105,6 +1105,17 @@ spec: description: Enabled defines if the Jaeger UI component should be created. type: boolean + findTracesConcurrentRequests: + description: |- + FindTracesConcurrentRequests defines how many concurrent request a single trace search can submit (defaults 2). + The search for traces in Jaeger submits limit+1 requests. First requests finds trace IDs and then it fetches + entire traces by ID. This property allows Jaeger to fetch traces in parallel. + Note that by default a single Tempo querier can process 20 concurrent search jobs. + Increasing this property might require scaling up querier instances, especially on error "job queue full" + See also Tempo's extraConfig: + querier.max_concurrent_queries (20 default) + query_frontend.max_outstanding_per_tenant: (2000 default). Increase if the query-frontend returns 429 + type: integer ingress: description: Ingress defines the Ingress configuration for the Jaeger UI. diff --git a/config/crd/bases/tempo.grafana.com_tempostacks.yaml b/config/crd/bases/tempo.grafana.com_tempostacks.yaml index f1374fbd5..6212f3001 100644 --- a/config/crd/bases/tempo.grafana.com_tempostacks.yaml +++ b/config/crd/bases/tempo.grafana.com_tempostacks.yaml @@ -2387,6 +2387,17 @@ spec: description: Enabled defines if the Jaeger Query component should be created. type: boolean + findTracesConcurrentRequests: + description: |- + FindTracesConcurrentRequests defines how many concurrent request a single trace search can submit (defaults querier.replicas*2). + The search for traces in Jaeger submits limit+1 requests. First requests finds trace IDs and then it fetches + entire traces by ID. This property allows Jaeger to fetch traces in parallel. + Note that by default a single Tempo querier can process 20 concurrent search jobs. + Increasing this property might require scaling up querier instances, especially on error "job queue full" + See also Tempo's extraConfig: + querier.max_concurrent_queries (20 default) + query_frontend.max_outstanding_per_tenant: (2000 default). Increase if the query-frontend returns 429 + type: integer ingress: description: Ingress defines the options for the Jaeger Query ingress. diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 4db4cd802..a1a7f323f 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -44,7 +44,7 @@ spec: - name: RELATED_IMAGE_JAEGER_QUERY value: docker.io/jaegertracing/jaeger-query:1.62.0 - name: RELATED_IMAGE_TEMPO_QUERY - value: docker.io/grafana/tempo-query:main-2999520 + value: docker.io/grafana/tempo-query:main-1de25ca - name: RELATED_IMAGE_TEMPO_GATEWAY value: quay.io/observatorium/api:main-2024-08-05-11d0d94 - name: RELATED_IMAGE_TEMPO_GATEWAY_OPA diff --git a/config/manifests/community/bases/tempo-operator.clusterserviceversion.yaml b/config/manifests/community/bases/tempo-operator.clusterserviceversion.yaml index 419a32f01..183b5d776 100644 --- a/config/manifests/community/bases/tempo-operator.clusterserviceversion.yaml +++ b/config/manifests/community/bases/tempo-operator.clusterserviceversion.yaml @@ -236,6 +236,19 @@ spec: "{"namespace": "", "resource": "pods", "verb": "get"}' displayName: SAR path: jaegerui.authentication.sar + - description: 'FindTracesConcurrentRequests defines how many concurrent request + a single trace search can submit (defaults 2). The search for traces in + Jaeger submits limit+1 requests. First requests finds trace IDs and then + it fetches entire traces by ID. This property allows Jaeger to fetch traces + in parallel. Note that by default a single Tempo querier can process 20 + concurrent search jobs. Increasing this property might require scaling up + querier instances, especially on error "job queue full" See also Tempo''s + extraConfig: querier.max_concurrent_queries (20 default) query_frontend.max_outstanding_per_tenant: + (2000 default). Increase if the query-frontend returns 429' + displayName: FindTracesConcurrentRequests + path: jaegerui.findTracesConcurrentRequests + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced - description: Annotations defines the annotations of the Ingress object. displayName: Annotations path: jaegerui.ingress.annotations @@ -931,6 +944,19 @@ spec: path: template.queryFrontend.jaegerQuery.enabled x-descriptors: - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - description: 'FindTracesConcurrentRequests defines how many concurrent request + a single trace search can submit (defaults querier.replicas*2). The search + for traces in Jaeger submits limit+1 requests. First requests finds trace + IDs and then it fetches entire traces by ID. This property allows Jaeger + to fetch traces in parallel. Note that by default a single Tempo querier + can process 20 concurrent search jobs. Increasing this property might require + scaling up querier instances, especially on error "job queue full" See also + Tempo''s extraConfig: querier.max_concurrent_queries (20 default) query_frontend.max_outstanding_per_tenant: + (2000 default). Increase if the query-frontend returns 429' + displayName: FindTracesConcurrentRequests + path: template.queryFrontend.jaegerQuery.findTracesConcurrentRequests + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced - description: Ingress defines the options for the Jaeger Query ingress. displayName: Jaeger Query UI Ingress Settings path: template.queryFrontend.jaegerQuery.ingress diff --git a/config/manifests/openshift/bases/tempo-operator.clusterserviceversion.yaml b/config/manifests/openshift/bases/tempo-operator.clusterserviceversion.yaml index 1b04fa519..55a0cd54a 100644 --- a/config/manifests/openshift/bases/tempo-operator.clusterserviceversion.yaml +++ b/config/manifests/openshift/bases/tempo-operator.clusterserviceversion.yaml @@ -236,6 +236,19 @@ spec: "{"namespace": "", "resource": "pods", "verb": "get"}' displayName: SAR path: jaegerui.authentication.sar + - description: 'FindTracesConcurrentRequests defines how many concurrent request + a single trace search can submit (defaults 2). The search for traces in + Jaeger submits limit+1 requests. First requests finds trace IDs and then + it fetches entire traces by ID. This property allows Jaeger to fetch traces + in parallel. Note that by default a single Tempo querier can process 20 + concurrent search jobs. Increasing this property might require scaling up + querier instances, especially on error "job queue full" See also Tempo''s + extraConfig: querier.max_concurrent_queries (20 default) query_frontend.max_outstanding_per_tenant: + (2000 default). Increase if the query-frontend returns 429' + displayName: FindTracesConcurrentRequests + path: jaegerui.findTracesConcurrentRequests + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced - description: Annotations defines the annotations of the Ingress object. displayName: Annotations path: jaegerui.ingress.annotations @@ -931,6 +944,19 @@ spec: path: template.queryFrontend.jaegerQuery.enabled x-descriptors: - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - description: 'FindTracesConcurrentRequests defines how many concurrent request + a single trace search can submit (defaults querier.replicas*2). The search + for traces in Jaeger submits limit+1 requests. First requests finds trace + IDs and then it fetches entire traces by ID. This property allows Jaeger + to fetch traces in parallel. Note that by default a single Tempo querier + can process 20 concurrent search jobs. Increasing this property might require + scaling up querier instances, especially on error "job queue full" See also + Tempo''s extraConfig: querier.max_concurrent_queries (20 default) query_frontend.max_outstanding_per_tenant: + (2000 default). Increase if the query-frontend returns 429' + displayName: FindTracesConcurrentRequests + path: template.queryFrontend.jaegerQuery.findTracesConcurrentRequests + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced - description: Ingress defines the options for the Jaeger Query ingress. displayName: Jaeger Query UI Ingress Settings path: template.queryFrontend.jaegerQuery.ingress diff --git a/docs/spec/tempo.grafana.com_tempomonolithics.yaml b/docs/spec/tempo.grafana.com_tempomonolithics.yaml index 487a6cfd7..f4ab9605b 100644 --- a/docs/spec/tempo.grafana.com_tempomonolithics.yaml +++ b/docs/spec/tempo.grafana.com_tempomonolithics.yaml @@ -35,6 +35,7 @@ spec: # TempoMonolithicSpec defines the desir requests: # Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. Requests cannot exceed Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ cpu: "500m" memory: "1Gi" + findTracesConcurrentRequests: 0 # FindTracesConcurrentRequests defines how many concurrent request a single trace search can submit (defaults 2). The search for traces in Jaeger submits limit+1 requests. First requests finds trace IDs and then it fetches entire traces by ID. This property allows Jaeger to fetch traces in parallel. Note that by default a single Tempo querier can process 20 concurrent search jobs. Increasing this property might require scaling up querier instances, especially on error "job queue full" See also Tempo's extraConfig: querier.max_concurrent_queries (20 default) query_frontend.max_outstanding_per_tenant: (2000 default). Increase if the query-frontend returns 429 ingress: # Ingress defines the Ingress configuration for the Jaeger UI. enabled: false # Enabled defines if an Ingress object should be created for Jaeger UI. annotations: {} # Annotations defines the annotations of the Ingress object. diff --git a/docs/spec/tempo.grafana.com_tempostacks.yaml b/docs/spec/tempo.grafana.com_tempostacks.yaml index 2b65cc6a7..ac32bdb41 100644 --- a/docs/spec/tempo.grafana.com_tempostacks.yaml +++ b/docs/spec/tempo.grafana.com_tempostacks.yaml @@ -330,6 +330,7 @@ spec: # TempoStackSpec defines the desired st requests: # Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. Requests cannot exceed Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ cpu: "500m" memory: "1Gi" + findTracesConcurrentRequests: 0 # FindTracesConcurrentRequests defines how many concurrent request a single trace search can submit (defaults querier.replicas*2). The search for traces in Jaeger submits limit+1 requests. First requests finds trace IDs and then it fetches entire traces by ID. This property allows Jaeger to fetch traces in parallel. Note that by default a single Tempo querier can process 20 concurrent search jobs. Increasing this property might require scaling up querier instances, especially on error "job queue full" See also Tempo's extraConfig: querier.max_concurrent_queries (20 default) query_frontend.max_outstanding_per_tenant: (2000 default). Increase if the query-frontend returns 429 ingress: # Ingress defines the options for the Jaeger Query ingress. annotations: {} # Annotations defines the annotations of the Ingress object. host: "" # Host defines the hostname of the Ingress object. diff --git a/internal/manifests/config/build.go b/internal/manifests/config/build.go index a20a60d78..9b17bc705 100644 --- a/internal/manifests/config/build.go +++ b/internal/manifests/config/build.go @@ -169,6 +169,14 @@ func buildTempoQueryConfig(params manifestutils.Params) ([]byte, error) { return []byte{}, err } + findTracesConcurrentRequests := params.Tempo.Spec.Template.QueryFrontend.JaegerQuery.FindTracesConcurrentRequests + if findTracesConcurrentRequests == 0 { + querierReplicas := int32(1) + if params.Tempo.Spec.Template.Querier.Replicas != nil { + querierReplicas = *params.Tempo.Spec.Template.Querier.Replicas + } + findTracesConcurrentRequests = int(querierReplicas) * 2 + } return renderTempoQueryTemplate(tempoQueryOptions{ TLS: tlsopts, HTTPPort: manifestutils.PortHTTPServer, @@ -176,9 +184,10 @@ func buildTempoQueryConfig(params manifestutils.Params) ([]byte, error) { GRPCEncryption: params.CtrlConfig.Gates.GRPCEncryption, HTTPEncryption: params.CtrlConfig.Gates.HTTPEncryption, }, - TenantHeader: manifestutils.TenantHeader, - Gateway: params.Tempo.Spec.Template.Gateway.Enabled, - ServicesQueryDuration: params.Tempo.Spec.Template.QueryFrontend.JaegerQuery.ServicesQueryDuration.Duration.String(), + TenantHeader: manifestutils.TenantHeader, + Gateway: params.Tempo.Spec.Template.Gateway.Enabled, + ServicesQueryDuration: params.Tempo.Spec.Template.QueryFrontend.JaegerQuery.ServicesQueryDuration.Duration.String(), + FindTracesConcurrentRequests: findTracesConcurrentRequests, }) } diff --git a/internal/manifests/config/options.go b/internal/manifests/config/options.go index 02ce67481..d34519764 100644 --- a/internal/manifests/config/options.go +++ b/internal/manifests/config/options.go @@ -27,12 +27,13 @@ type options struct { } type tempoQueryOptions struct { - Gates featureGates - TLS tlsOptions - HTTPPort int - TenantHeader string - Gateway bool - ServicesQueryDuration string + Gates featureGates + TLS tlsOptions + HTTPPort int + TenantHeader string + Gateway bool + ServicesQueryDuration string + FindTracesConcurrentRequests int } type featureGates struct { diff --git a/internal/manifests/config/tempo-query.yaml b/internal/manifests/config/tempo-query.yaml index 4096bc107..782c95d3d 100644 --- a/internal/manifests/config/tempo-query.yaml +++ b/internal/manifests/config/tempo-query.yaml @@ -10,3 +10,4 @@ tls_insecure_skip_verify: false tls_server_name: {{ .TLS.ServerNames.QueryFrontend }} {{- end }} services_query_duration: {{ .ServicesQueryDuration }} +find_traces_concurrent_requests: {{ .FindTracesConcurrentRequests }} diff --git a/internal/manifests/monolithic/configmap.go b/internal/manifests/monolithic/configmap.go index 8c1a1ea94..c9ae7ba2f 100644 --- a/internal/manifests/monolithic/configmap.go +++ b/internal/manifests/monolithic/configmap.go @@ -96,10 +96,11 @@ type tempoConfig struct { } type tempoQueryConfig struct { - Address string `yaml:"address"` - Backend string `yaml:"backend"` - TenantHeaderKey string `yaml:"tenant_header_key"` - ServicesQueryDuration time.Duration `yaml:"services_query_duration"` + Address string `yaml:"address"` + Backend string `yaml:"backend"` + TenantHeaderKey string `yaml:"tenant_header_key"` + ServicesQueryDuration time.Duration `yaml:"services_query_duration"` + FindTracesConcurrentRequests int `yaml:"find_traces_concurrent_requests"` } // BuildConfigMap creates the Tempo ConfigMap for a monolithic deployment. @@ -294,5 +295,6 @@ func buildTempoQueryConfig(jaegerUISpec *v1alpha1.MonolithicJaegerUISpec) ([]byt config.Backend = fmt.Sprintf("127.0.0.1:%d", manifestutils.PortHTTPServer) config.TenantHeaderKey = manifestutils.TenantHeader config.ServicesQueryDuration = jaegerUISpec.ServicesQueryDuration.Duration + config.FindTracesConcurrentRequests = jaegerUISpec.FindTracesConcurrentRequests return yaml.Marshal(&config) } diff --git a/internal/manifests/monolithic/configmap_test.go b/internal/manifests/monolithic/configmap_test.go index 7eab5b3f3..79ed15f45 100644 --- a/internal/manifests/monolithic/configmap_test.go +++ b/internal/manifests/monolithic/configmap_test.go @@ -42,8 +42,9 @@ func TestBuildConfigMap(t *testing.T) { }, }, JaegerUI: &v1alpha1.MonolithicJaegerUISpec{ - Enabled: true, - ServicesQueryDuration: &metav1.Duration{Duration: time.Duration(3 * 24 * time.Hour)}, + Enabled: true, + ServicesQueryDuration: &metav1.Duration{Duration: time.Duration(3 * 24 * time.Hour)}, + FindTracesConcurrentRequests: 22, }, }, }, @@ -63,6 +64,7 @@ address: 127.0.0.1:7777 backend: 127.0.0.1:3200 tenant_header_key: x-scope-orgid services_query_duration: 72h0m0s +find_traces_concurrent_requests: 22 ` require.YAMLEq(t, tempoQueryCfg, cm.Data["tempo-query.yaml"]) } diff --git a/tests/e2e/monolithic-extraconfig/install-tempo-assert.yaml b/tests/e2e/monolithic-extraconfig/install-tempo-assert.yaml index b2244e1df..883e687d7 100644 --- a/tests/e2e/monolithic-extraconfig/install-tempo-assert.yaml +++ b/tests/e2e/monolithic-extraconfig/install-tempo-assert.yaml @@ -81,6 +81,7 @@ data: backend: 127.0.0.1:3200 tenant_header_key: x-scope-orgid services_query_duration: 72h0m0s + find_traces_concurrent_requests: 2 tempo.yaml: | distributor: receivers: diff --git a/tests/e2e/tempostack-extraconfig/install-tempostack-assert.yaml b/tests/e2e/tempostack-extraconfig/install-tempostack-assert.yaml index 4cad151d3..6b7d3b59c 100644 --- a/tests/e2e/tempostack-extraconfig/install-tempostack-assert.yaml +++ b/tests/e2e/tempostack-extraconfig/install-tempostack-assert.yaml @@ -480,6 +480,7 @@ data: backend: 127.0.0.1:3200 tenant_header_key: x-scope-orgid services_query_duration: 72h0m0s + find_traces_concurrent_requests: 2 tempo.yaml: | compactor: compaction: From 3084bc9191af729458ae9a95e5a6793fdc8d97a4 Mon Sep 17 00:00:00 2001 From: Israel Blancas Date: Fri, 11 Oct 2024 12:23:20 +0200 Subject: [PATCH 09/10] Fix default value for jaeger query ingress type (#1055) * Fix default value for jaeger query ingress type Signed-off-by: Israel Blancas * Fix conflict Signed-off-by: Israel Blancas --------- Signed-off-by: Israel Blancas --- .chloggen/fix-ingress-type-default.yaml | 16 ++++++++++++++++ apis/tempo/v1alpha1/ingress_types.go | 3 ++- .../manifests/tempo.grafana.com_tempostacks.yaml | 2 ++ .../manifests/tempo.grafana.com_tempostacks.yaml | 2 ++ .../crd/bases/tempo.grafana.com_tempostacks.yaml | 2 ++ 5 files changed, 24 insertions(+), 1 deletion(-) create mode 100755 .chloggen/fix-ingress-type-default.yaml diff --git a/.chloggen/fix-ingress-type-default.yaml b/.chloggen/fix-ingress-type-default.yaml new file mode 100755 index 000000000..7097693cc --- /dev/null +++ b/.chloggen/fix-ingress-type-default.yaml @@ -0,0 +1,16 @@ +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: bug_fix + +# The name of the component, or a single word describing the area of concern, (e.g. tempostack, tempomonolithic, github action) +component: tempostack + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: The default value for the IngressType type is now correctly "" (empty string). Previously, it was impossible to select it in tools like the OpenShift web console, what could cause some issues. + +# One or more tracking issues related to the change +issues: [1054] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: diff --git a/apis/tempo/v1alpha1/ingress_types.go b/apis/tempo/v1alpha1/ingress_types.go index 75f51117b..439edd80a 100644 --- a/apis/tempo/v1alpha1/ingress_types.go +++ b/apis/tempo/v1alpha1/ingress_types.go @@ -2,7 +2,8 @@ package v1alpha1 type ( // IngressType represents how a service should be exposed (ingress vs route). - // +kubebuilder:validation:Enum=ingress;route + // +kubebuilder:validation:Enum=ingress;route;"" + // +kubebuilder:default="" IngressType string ) diff --git a/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml b/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml index a34011a10..6ec9eaffb 100644 --- a/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml +++ b/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml @@ -1431,6 +1431,7 @@ spec: enum: - ingress - route + - "" type: string type: object required: @@ -2443,6 +2444,7 @@ spec: enum: - ingress - route + - "" type: string type: object monitorTab: diff --git a/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml b/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml index a34011a10..6ec9eaffb 100644 --- a/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml +++ b/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml @@ -1431,6 +1431,7 @@ spec: enum: - ingress - route + - "" type: string type: object required: @@ -2443,6 +2444,7 @@ spec: enum: - ingress - route + - "" type: string type: object monitorTab: diff --git a/config/crd/bases/tempo.grafana.com_tempostacks.yaml b/config/crd/bases/tempo.grafana.com_tempostacks.yaml index 6212f3001..9b156e754 100644 --- a/config/crd/bases/tempo.grafana.com_tempostacks.yaml +++ b/config/crd/bases/tempo.grafana.com_tempostacks.yaml @@ -1427,6 +1427,7 @@ spec: enum: - ingress - route + - "" type: string type: object required: @@ -2439,6 +2440,7 @@ spec: enum: - ingress - route + - "" type: string type: object monitorTab: From 5238c4c28bee1ab5619bab0999ce164674e61f88 Mon Sep 17 00:00:00 2001 From: Ishwar Kanse Date: Fri, 11 Oct 2024 19:45:16 +0530 Subject: [PATCH 10/10] Fix e2e tests (#1056) --- .../scale-tempo-assert.yaml | 3 +- .../03-assert.yaml | 22 ++++++++ .../03-install-otel.yaml | 51 +++---------------- .../monolithic-route/check-must-gahter.sh | 4 +- .../install-tempo-assert.yaml | 1 + .../e2e-openshift/route/check-must-gahter.sh | 3 +- .../route/install-tempo-assert.yaml | 1 + .../install-tempo-assert.yaml | 48 +++++++++++++++-- .../01-assert.yaml | 1 + .../tls-singletenant/01-assert.yaml | 40 +++++++++++++-- 10 files changed, 117 insertions(+), 57 deletions(-) diff --git a/tests/e2e-openshift/component-replicas/scale-tempo-assert.yaml b/tests/e2e-openshift/component-replicas/scale-tempo-assert.yaml index 2d07ffb5d..3e493fa8d 100644 --- a/tests/e2e-openshift/component-replicas/scale-tempo-assert.yaml +++ b/tests/e2e-openshift/component-replicas/scale-tempo-assert.yaml @@ -135,6 +135,7 @@ metadata: kind: TempoStack name: cmpreps spec: + replicas: 2 selector: matchLabels: app.kubernetes.io/component: gateway @@ -156,8 +157,8 @@ spec: - --web.internal.listen=0.0.0.0:8081 - --traces.write.otlpgrpc.endpoint=tempo-cmpreps-distributor.chainsaw-replicas.svc.cluster.local:4317 - --traces.write.otlphttp.endpoint=https://tempo-cmpreps-distributor.chainsaw-replicas.svc.cluster.local:4318 - - --traces.tempo.endpoint=https://tempo-cmpreps-query-frontend.chainsaw-replicas.svc.cluster.local:3200 - --traces.write-timeout=30s + - --traces.tempo.endpoint=https://tempo-cmpreps-query-frontend.chainsaw-replicas.svc.cluster.local:3200 - --grpc.listen=0.0.0.0:8090 - --rbac.config=/etc/tempo-gateway/cm/rbac.yaml - --tenants.config=/etc/tempo-gateway/secret/tenants.yaml diff --git a/tests/e2e-openshift/monolithic-multitenancy-static/03-assert.yaml b/tests/e2e-openshift/monolithic-multitenancy-static/03-assert.yaml index 0d3eaf2bc..b1e2dff23 100644 --- a/tests/e2e-openshift/monolithic-multitenancy-static/03-assert.yaml +++ b/tests/e2e-openshift/monolithic-multitenancy-static/03-assert.yaml @@ -2,5 +2,27 @@ apiVersion: apps/v1 kind: Deployment metadata: name: opentelemetry-collector + namespace: chainsaw-monolithic-multitenancy-static status: + availableReplicas: 1 readyReplicas: 1 + replicas: 1 + +--- +apiVersion: v1 +kind: Service +metadata: + name: opentelemetry-collector + namespace: chainsaw-monolithic-multitenancy-static +spec: + ports: + - appProtocol: grpc + name: otlp-grpc + port: 4317 + protocol: TCP + targetPort: 4317 + selector: + app.kubernetes.io/component: opentelemetry-collector + app.kubernetes.io/instance: chainsaw-monolithic-multitenancy-static.opentelemetry + app.kubernetes.io/managed-by: opentelemetry-operator + app.kubernetes.io/part-of: opentelemetry \ No newline at end of file diff --git a/tests/e2e-openshift/monolithic-multitenancy-static/03-install-otel.yaml b/tests/e2e-openshift/monolithic-multitenancy-static/03-install-otel.yaml index b9ed7c52d..3f97a5dca 100644 --- a/tests/e2e-openshift/monolithic-multitenancy-static/03-install-otel.yaml +++ b/tests/e2e-openshift/monolithic-multitenancy-static/03-install-otel.yaml @@ -1,9 +1,10 @@ -apiVersion: v1 -kind: ConfigMap +apiVersion: opentelemetry.io/v1alpha1 +kind: OpenTelemetryCollector metadata: - name: opentelemetry-collector-configmap -data: - config.yaml: | + name: opentelemetry +spec: + mode: deployment + config: | extensions: oauth2client: client_id: tenant1-oidc-client @@ -35,42 +36,4 @@ data: pipelines: traces: exporters: [otlp] - receivers: [otlp] ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: opentelemetry-collector -spec: - selector: - matchLabels: - app: opentelemetry-collector - template: - metadata: - labels: - app: opentelemetry-collector - spec: - containers: - - name: opentelemetry-collector - image: ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib:0.106.1 - command: ["/otelcol-contrib", "--config=/conf/config.yaml"] - volumeMounts: - - mountPath: /conf - name: opentelemetry-collector-configmap - volumes: - - name: opentelemetry-collector-configmap - configMap: - name: opentelemetry-collector-configmap ---- -apiVersion: v1 -kind: Service -metadata: - name: opentelemetry-collector -spec: - type: ClusterIP - ports: - - name: otlp-grpc - port: 4317 - targetPort: 4317 - selector: - app: opentelemetry-collector + receivers: [otlp] \ No newline at end of file diff --git a/tests/e2e-openshift/monolithic-route/check-must-gahter.sh b/tests/e2e-openshift/monolithic-route/check-must-gahter.sh index 1cba22ef1..2757209a3 100755 --- a/tests/e2e-openshift/monolithic-route/check-must-gahter.sh +++ b/tests/e2e-openshift/monolithic-route/check-must-gahter.sh @@ -12,12 +12,10 @@ REQUIRED_ITEMS=( "event-filter.html" "timestamp" "*sha*/deployment-tempo-operator-controller.yaml" - "*sha*/olm/operator-servicemeshoperator-openshift-operators.yaml" "*sha*/olm/installplan-install-*.yaml" "*sha*/olm/clusterserviceversion-tempo-operator-*.yaml" "*sha*/olm/operator-opentelemetry-product-openshift-opentelemetry-operator.yaml" - "*sha*/olm/operator-tempo-operator-tempo-operator.yaml" - "*sha*/olm/operator-tempo-product-openshift-tempo-operator.yaml" + "*sha*/olm/operator-tempo-*-tempo-operator.yaml" "*sha*/olm/subscription-tempo-operator-*-sub.yaml" "*sha*/namespaces/chainsaw-mono-route/tempomonolithic/mono-route/tempomonolithic-mono-route.yaml" "*sha*/namespaces/chainsaw-mono-route/tempomonolithic/mono-route/service-tempo-mono-route-jaegerui.yaml" diff --git a/tests/e2e-openshift/monolithic-route/install-tempo-assert.yaml b/tests/e2e-openshift/monolithic-route/install-tempo-assert.yaml index 07874955c..b82463b8f 100644 --- a/tests/e2e-openshift/monolithic-route/install-tempo-assert.yaml +++ b/tests/e2e-openshift/monolithic-route/install-tempo-assert.yaml @@ -258,6 +258,7 @@ data: backend: 127.0.0.1:3200 tenant_header_key: x-scope-orgid services_query_duration: 72h0m0s + find_traces_concurrent_requests: 2 tempo.yaml: | server: http_listen_port: 3200 diff --git a/tests/e2e-openshift/route/check-must-gahter.sh b/tests/e2e-openshift/route/check-must-gahter.sh index 736dff03f..fb3b1b1b8 100755 --- a/tests/e2e-openshift/route/check-must-gahter.sh +++ b/tests/e2e-openshift/route/check-must-gahter.sh @@ -15,8 +15,7 @@ REQUIRED_ITEMS=( "*sha*/olm/installplan-install-*" "*sha*/olm/clusterserviceversion-tempo-operator-*.yaml" "*sha*/olm/operator-opentelemetry-product-openshift-opentelemetry-operator.yaml" - "*sha*/olm/operator-tempo-operator-tempo-operator.yaml" - "*sha*/olm/operator-tempo-product-openshift-tempo-operator.yaml" + "*sha*/olm/operator-*-tempo-operator.yaml" "*sha*/olm/subscription-tempo-operator-*-sub.yaml" "*sha*/namespaces/chainsaw-route/tempostack/simplest/service-tempo-simplest-distributor.yaml" "*sha*/namespaces/chainsaw-route/tempostack/simplest/service-tempo-simplest-ingester.yaml" diff --git a/tests/e2e-openshift/route/install-tempo-assert.yaml b/tests/e2e-openshift/route/install-tempo-assert.yaml index 8fba41e72..dfe8e64c0 100644 --- a/tests/e2e-openshift/route/install-tempo-assert.yaml +++ b/tests/e2e-openshift/route/install-tempo-assert.yaml @@ -129,6 +129,7 @@ data: backend: 127.0.0.1:3200 tenant_header_key: x-scope-orgid services_query_duration: 72h0m0s + find_traces_concurrent_requests: 2 tempo.yaml: | compactor: compaction: diff --git a/tests/e2e-openshift/tempo-single-tenant-auth/install-tempo-assert.yaml b/tests/e2e-openshift/tempo-single-tenant-auth/install-tempo-assert.yaml index 40f3dcafe..9b6f74c5b 100644 --- a/tests/e2e-openshift/tempo-single-tenant-auth/install-tempo-assert.yaml +++ b/tests/e2e-openshift/tempo-single-tenant-auth/install-tempo-assert.yaml @@ -47,7 +47,25 @@ spec: tempo-gossip-member: "true" spec: containers: - - name: tempo + - args: + - -target=query-frontend + - -config.file=/conf/tempo-query-frontend.yaml + - -mem-ballast-size-mbs=1024 + - -log.level=info + - --storage.trace.s3.secret_key=$(S3_SECRET_KEY) + - --storage.trace.s3.access_key=$(S3_ACCESS_KEY) + env: + - name: S3_SECRET_KEY + valueFrom: + secretKeyRef: + key: access_key_secret + name: minio + - name: S3_ACCESS_KEY + valueFrom: + secretKeyRef: + key: access_key_id + name: minio + name: tempo ports: - containerPort: 3200 name: http @@ -65,7 +83,12 @@ spec: name: tempo-tempo-st-ca-bundle - mountPath: /var/run/tls/server name: tempo-tempo-st-query-frontend-mtls - - name: jaeger-query + - args: + - --query.base-path=/ + - --span-storage.type=grpc + - --grpc-storage.server=localhost:7777 + - --query.bearer-token-propagation=true + name: jaeger-query ports: - containerPort: 16685 name: jaeger-grpc @@ -79,7 +102,13 @@ spec: volumeMounts: - mountPath: /tmp name: tempo-tmp-storage-query - - name: tempo-query + - mountPath: /var/run/ca + name: tempo-tempo-st-ca-bundle + - mountPath: /var/run/tls/server + name: tempo-tempo-st-query-frontend-mtls + - args: + - -config=/conf/tempo-query.yaml + name: tempo-query ports: - containerPort: 7777 name: proxy-grpc @@ -92,7 +121,18 @@ spec: name: tempo-tempo-st-ca-bundle - mountPath: /var/run/tls/server name: tempo-tempo-st-query-frontend-mtls - - name: oauth-proxy + - args: + - --cookie-secret-file=/var/run/secrets/kubernetes.io/serviceaccount/token + - --https-address=:8443 + - --openshift-service-account=tempo-tempo-st-query-frontend + - --provider=openshift + - --tls-cert=/etc/tls/private/tls.crt + - --tls-key=/etc/tls/private/tls.key + - --upstream=http://localhost:16686 + - --upstream-timeout=30s + - '--openshift-sar={"namespace": "chainsaw-mst", "resource": "pods", "verb": + "get"}' + name: oauth-proxy ports: - containerPort: 8443 name: oauth-proxy diff --git a/tests/e2e-openshift/tls-monolithic-singletenant/01-assert.yaml b/tests/e2e-openshift/tls-monolithic-singletenant/01-assert.yaml index 3ea628016..c882016ee 100644 --- a/tests/e2e-openshift/tls-monolithic-singletenant/01-assert.yaml +++ b/tests/e2e-openshift/tls-monolithic-singletenant/01-assert.yaml @@ -302,6 +302,7 @@ data: backend: 127.0.0.1:3200 tenant_header_key: x-scope-orgid services_query_duration: 72h0m0s + find_traces_concurrent_requests: 2 tempo.yaml: | server: http_listen_port: 3200 diff --git a/tests/e2e-openshift/tls-singletenant/01-assert.yaml b/tests/e2e-openshift/tls-singletenant/01-assert.yaml index 523f74e6a..b2202982a 100644 --- a/tests/e2e-openshift/tls-singletenant/01-assert.yaml +++ b/tests/e2e-openshift/tls-singletenant/01-assert.yaml @@ -42,7 +42,25 @@ spec: tempo-gossip-member: "true" spec: containers: - - name: tempo + - args: + - -target=query-frontend + - -config.file=/conf/tempo-query-frontend.yaml + - -mem-ballast-size-mbs=1024 + - -log.level=info + - --storage.trace.s3.secret_key=$(S3_SECRET_KEY) + - --storage.trace.s3.access_key=$(S3_ACCESS_KEY) + env: + - name: S3_SECRET_KEY + valueFrom: + secretKeyRef: + key: access_key_secret + name: minio + - name: S3_ACCESS_KEY + valueFrom: + secretKeyRef: + key: access_key_id + name: minio + name: tempo ports: - containerPort: 3200 name: http @@ -60,7 +78,12 @@ spec: name: tempo-simplest-ca-bundle - mountPath: /var/run/tls/server name: tempo-simplest-query-frontend-mtls - - name: jaeger-query + - args: + - --query.base-path=/ + - --span-storage.type=grpc + - --grpc-storage.server=localhost:7777 + - --query.bearer-token-propagation=true + name: jaeger-query ports: - containerPort: 16685 name: jaeger-grpc @@ -74,7 +97,13 @@ spec: volumeMounts: - mountPath: /tmp name: tempo-tmp-storage-query - - name: tempo-query + - mountPath: /var/run/ca + name: tempo-simplest-ca-bundle + - mountPath: /var/run/tls/server + name: tempo-simplest-query-frontend-mtls + - args: + - -config=/conf/tempo-query.yaml + name: tempo-query ports: - containerPort: 7777 name: proxy-grpc @@ -99,6 +128,10 @@ spec: - '--openshift-sar={"namespace": "chainsaw-tls-singletenant", "resource": "pods", "verb": "get"}' name: oauth-proxy + ports: + - containerPort: 8443 + name: oauth-proxy + protocol: TCP volumes: - configMap: defaultMode: 420 @@ -195,6 +228,7 @@ data: backend: 127.0.0.1:3200 tenant_header_key: x-scope-orgid services_query_duration: 72h0m0s + find_traces_concurrent_requests: 2 tempo.yaml: "compactor:\n compaction:\n block_retention: 48h0m0s\n ring:\n \ kvstore:\n store: memberlist\ndistributor:\n receivers:\n jaeger:\n \ protocols:\n thrift_http:\n endpoint: 0.0.0.0:14268\n tls:\n