diff --git a/.chloggen/bump_jaeger-to-1.62.yaml b/.chloggen/bump_jaeger-to-1.62.yaml new file mode 100755 index 000000000..96aa8c82d --- /dev/null +++ b/.chloggen/bump_jaeger-to-1.62.yaml @@ -0,0 +1,16 @@ +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. tempostack, tempomonolithic, github action) +component: tempostack, tempomonolithic + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: bump jaeger to v1.62 + +# One or more tracking issues related to the change +issues: [1050] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: diff --git a/.chloggen/fix-ingress-type-default.yaml b/.chloggen/fix-ingress-type-default.yaml new file mode 100755 index 000000000..7097693cc --- /dev/null +++ b/.chloggen/fix-ingress-type-default.yaml @@ -0,0 +1,16 @@ +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: bug_fix + +# The name of the component, or a single word describing the area of concern, (e.g. tempostack, tempomonolithic, github action) +component: tempostack + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: The default value for the IngressType type is now correctly "" (empty string). Previously, it was impossible to select it in tools like the OpenShift web console, what could cause some issues. + +# One or more tracking issues related to the change +issues: [1054] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: diff --git a/.chloggen/fix_jaeger-query-certs.yaml b/.chloggen/fix_jaeger-query-certs.yaml new file mode 100755 index 000000000..d1ecc3405 --- /dev/null +++ b/.chloggen/fix_jaeger-query-certs.yaml @@ -0,0 +1,16 @@ +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: bug_fix + +# The name of the component, or a single word describing the area of concern, (e.g. tempostack, tempomonolithic, github action) +component: tempostack + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: grant jaeer-query access to pki certs + +# One or more tracking issues related to the change +issues: [1051] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: diff --git a/.chloggen/tempo-query-find-traces-jobs.yaml b/.chloggen/tempo-query-find-traces-jobs.yaml new file mode 100755 index 000000000..294deef9f --- /dev/null +++ b/.chloggen/tempo-query-find-traces-jobs.yaml @@ -0,0 +1,21 @@ +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. tempostack, tempomonolithic, github action) +component: tempostack, tempomonolithic + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Add tempo-query CRD option to speed up trace search. + +# One or more tracking issues related to the change +issues: [1048] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: | + Following CRD options were added to speed up trace search in Jaeger UI/API. The trace search first + searches for traceids and then it gets a full trace. With this configuration option the requests + to get the full trace can be run in parallel: + For `TempoStack` - `spec.template.queryFrontend.jaegerQuery.findTracesConcurrentRequests` + For `TempoMonolithic` - `spec.jaegerui.findTracesConcurrentRequests` diff --git a/.chloggen/timeout.yaml b/.chloggen/timeout.yaml new file mode 100755 index 000000000..e8de1bf44 --- /dev/null +++ b/.chloggen/timeout.yaml @@ -0,0 +1,18 @@ +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: breaking + +# The name of the component, or a single word describing the area of concern, (e.g. tempostack, tempomonolithic, github action) +component: tempostack, tempomonolithic + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Add unified timeout configuration. It changes the default to 30s. + +# One or more tracking issues related to the change +issues: [1045] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: | + Adding `spec.timeout` CRD option to configure timeout on all components and default it to 30s. + Before Tempo server was defaulting to 3m, gateway to 2m, OpenShift route to 30s (for query), oauth-proxy to 30s (for query). diff --git a/Makefile b/Makefile index 2c91dff93..f5f488fff 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,11 @@ # Current Operator version OPERATOR_VERSION ?= 0.13.0 TEMPO_VERSION ?= 2.6.0 -TEMPO_QUERY_VERSION ?= main-2999520 -JAEGER_QUERY_VERSION ?= 1.60 +JAEGER_QUERY_VERSION ?= 1.62.0 +TEMPO_QUERY_VERSION ?= main-1de25ca TEMPO_GATEWAY_VERSION ?= main-2024-08-05-11d0d94 TEMPO_GATEWAY_OPA_VERSION ?= main-2024-04-29-914c13f -OAUTH_PROXY_VERSION=4.12 +OAUTH_PROXY_VERSION=4.14 MIN_KUBERNETES_VERSION ?= 1.25.0 MIN_OPENSHIFT_VERSION ?= 4.12 diff --git a/apis/tempo/v1alpha1/ingress_types.go b/apis/tempo/v1alpha1/ingress_types.go index 75f51117b..439edd80a 100644 --- a/apis/tempo/v1alpha1/ingress_types.go +++ b/apis/tempo/v1alpha1/ingress_types.go @@ -2,7 +2,8 @@ package v1alpha1 type ( // IngressType represents how a service should be exposed (ingress vs route). - // +kubebuilder:validation:Enum=ingress;route + // +kubebuilder:validation:Enum=ingress;route;"" + // +kubebuilder:default="" IngressType string ) diff --git a/apis/tempo/v1alpha1/tempomonolithic_defaults.go b/apis/tempo/v1alpha1/tempomonolithic_defaults.go index 7dfbd738e..148422d60 100644 --- a/apis/tempo/v1alpha1/tempomonolithic_defaults.go +++ b/apis/tempo/v1alpha1/tempomonolithic_defaults.go @@ -13,9 +13,11 @@ import ( ) var ( - twoGBQuantity = resource.MustParse("2Gi") - tenGBQuantity = resource.MustParse("10Gi") - defaultServicesDuration = metav1.Duration{Duration: time.Hour * 24 * 3} + twoGBQuantity = resource.MustParse("2Gi") + tenGBQuantity = resource.MustParse("10Gi") + defaultServicesDuration = metav1.Duration{Duration: time.Hour * 24 * 3} + defaultTimeout = metav1.Duration{Duration: time.Second * 30} + defaultFindTracesConcurrentRequests = 2 ) // Default sets all default values in a central place, instead of setting it at every place where the value is accessed. @@ -87,5 +89,12 @@ func (r *TempoMonolithic) Default(ctrlConfig configv1alpha1.ProjectConfig) { if r.Spec.JaegerUI.ServicesQueryDuration == nil { r.Spec.JaegerUI.ServicesQueryDuration = &defaultServicesDuration } + if r.Spec.JaegerUI.FindTracesConcurrentRequests == 0 { + r.Spec.JaegerUI.FindTracesConcurrentRequests = defaultFindTracesConcurrentRequests + } + } + + if r.Spec.Timeout.Duration == 0 { + r.Spec.Timeout = defaultTimeout } } diff --git a/apis/tempo/v1alpha1/tempomonolithic_defaults_test.go b/apis/tempo/v1alpha1/tempomonolithic_defaults_test.go index c41828ef6..0768c3688 100644 --- a/apis/tempo/v1alpha1/tempomonolithic_defaults_test.go +++ b/apis/tempo/v1alpha1/tempomonolithic_defaults_test.go @@ -6,6 +6,7 @@ import ( "github.com/stretchr/testify/assert" "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" configv1alpha1 "github.com/grafana/tempo-operator/apis/config/v1alpha1" @@ -46,6 +47,7 @@ func TestMonolithicDefault(t *testing.T) { }, }, Management: "Managed", + Timeout: metav1.Duration{Duration: time.Second * 30}, }, }, }, @@ -79,6 +81,7 @@ func TestMonolithicDefault(t *testing.T) { }, }, Management: "Managed", + Timeout: metav1.Duration{Duration: time.Second * 30}, }, }, }, @@ -104,6 +107,7 @@ func TestMonolithicDefault(t *testing.T) { }, }, Management: "Unmanaged", + Timeout: metav1.Duration{Duration: time.Second * 30}, }, }, expected: &TempoMonolithic{ @@ -125,6 +129,7 @@ func TestMonolithicDefault(t *testing.T) { }, }, Management: "Unmanaged", + Timeout: metav1.Duration{Duration: time.Second * 30}, }, }, }, @@ -191,9 +196,11 @@ func TestMonolithicDefault(t *testing.T) { Enabled: true, SAR: "{\"namespace\": \"testns\", \"resource\": \"pods\", \"verb\": \"get\"}", }, - ServicesQueryDuration: &defaultServicesDuration, + ServicesQueryDuration: &defaultServicesDuration, + FindTracesConcurrentRequests: 2, }, Management: "Managed", + Timeout: metav1.Duration{Duration: time.Second * 30}, }, }, }, @@ -263,9 +270,11 @@ func TestMonolithicDefault(t *testing.T) { Enabled: false, SAR: "{\"namespace\": \"testns\", \"resource\": \"pods\", \"verb\": \"get\"}", }, - ServicesQueryDuration: &defaultServicesDuration, + ServicesQueryDuration: &defaultServicesDuration, + FindTracesConcurrentRequests: 2, }, Management: "Managed", + Timeout: metav1.Duration{Duration: time.Second * 30}, }, }, }, @@ -327,9 +336,11 @@ func TestMonolithicDefault(t *testing.T) { Enabled: true, SAR: "{\"namespace\": \"testns\", \"resource\": \"pods\", \"verb\": \"get\"}", }, - ServicesQueryDuration: &defaultServicesDuration, + ServicesQueryDuration: &defaultServicesDuration, + FindTracesConcurrentRequests: 2, }, Management: "Managed", + Timeout: metav1.Duration{Duration: time.Second * 30}, }, }, }, @@ -390,14 +401,16 @@ func TestMonolithicDefault(t *testing.T) { Enabled: false, SAR: "{\"namespace\": \"testns\", \"resource\": \"pods\", \"verb\": \"get\"}", }, - ServicesQueryDuration: &defaultServicesDuration, + ServicesQueryDuration: &defaultServicesDuration, + FindTracesConcurrentRequests: 2, }, Management: "Managed", + Timeout: metav1.Duration{Duration: time.Second * 30}, }, }, }, { - name: "define custom duration for services list", + name: "define custom duration for services list, timeout and find traces", input: &TempoMonolithic{ ObjectMeta: v1.ObjectMeta{ Name: "test", @@ -415,8 +428,10 @@ func TestMonolithicDefault(t *testing.T) { Route: &MonolithicJaegerUIRouteSpec{ Enabled: true, }, - ServicesQueryDuration: &v1.Duration{Duration: time.Duration(100 * 100)}, + ServicesQueryDuration: &v1.Duration{Duration: time.Duration(100 * 100)}, + FindTracesConcurrentRequests: 40, }, + Timeout: metav1.Duration{Duration: time.Hour}, }, }, expected: &TempoMonolithic{ @@ -451,9 +466,11 @@ func TestMonolithicDefault(t *testing.T) { Enabled: false, SAR: "{\"namespace\": \"testns\", \"resource\": \"pods\", \"verb\": \"get\"}", }, - ServicesQueryDuration: &v1.Duration{Duration: time.Duration(100 * 100)}, + ServicesQueryDuration: &v1.Duration{Duration: time.Duration(100 * 100)}, + FindTracesConcurrentRequests: 40, }, Management: "Managed", + Timeout: metav1.Duration{Duration: time.Hour}, }, }, }, diff --git a/apis/tempo/v1alpha1/tempomonolithic_types.go b/apis/tempo/v1alpha1/tempomonolithic_types.go index 1d56c4400..b6db8fe0a 100644 --- a/apis/tempo/v1alpha1/tempomonolithic_types.go +++ b/apis/tempo/v1alpha1/tempomonolithic_types.go @@ -44,6 +44,11 @@ type TempoMonolithicSpec struct { // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Resources",order=5,xDescriptors="urn:alm:descriptor:com.tectonic.ui:resourceRequirements" Resources *corev1.ResourceRequirements `json:"resources,omitempty"` + // Timeout configures the same timeout on all components starting at ingress down to the ingestor/querier. + // Timeout configuration on a specific component has a higher precedence. + // Default is 30 seconds. + Timeout metav1.Duration `json:"timeout,omitempty"` + // ServiceAccount defines the Service Account to use for all Tempo components. // // +kubebuilder:validation:Optional @@ -256,6 +261,19 @@ type MonolithicJaegerUISpec struct { // +optional // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="ServicesQueryDuration",xDescriptors="urn:alm:descriptor:com.tectonic.ui:advanced" ServicesQueryDuration *metav1.Duration `json:"servicesQueryDuration,omitempty"` + + // FindTracesConcurrentRequests defines how many concurrent request a single trace search can submit (defaults 2). + // The search for traces in Jaeger submits limit+1 requests. First requests finds trace IDs and then it fetches + // entire traces by ID. This property allows Jaeger to fetch traces in parallel. + // Note that by default a single Tempo querier can process 20 concurrent search jobs. + // Increasing this property might require scaling up querier instances, especially on error "job queue full" + // See also Tempo's extraConfig: + // querier.max_concurrent_queries (20 default) + // query_frontend.max_outstanding_per_tenant: (2000 default). Increase if the query-frontend returns 429 + // + // +optional + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="FindTracesConcurrentRequests",xDescriptors="urn:alm:descriptor:com.tectonic.ui:advanced" + FindTracesConcurrentRequests int `json:"findTracesConcurrentRequests,omitempty"` } // MonolithicJaegerUIIngressSpec defines the settings for the Jaeger UI ingress. diff --git a/apis/tempo/v1alpha1/tempostack_types.go b/apis/tempo/v1alpha1/tempostack_types.go index c42fbb536..e868b207d 100644 --- a/apis/tempo/v1alpha1/tempostack_types.go +++ b/apis/tempo/v1alpha1/tempostack_types.go @@ -40,6 +40,11 @@ type TempoStackSpec struct { // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Ingestion and Querying Ratelimiting" LimitSpec LimitSpec `json:"limits,omitempty"` + // Timeout configures the same timeout on all components starting at ingress down to the ingestor/querier. + // Timeout configuration on a specific component has a higher precedence. + // Defaults to 30 seconds. + Timeout metav1.Duration `json:"timeout,omitempty"` + // StorageClassName for PVCs used by ingester. Defaults to nil (default storage class in the cluster). // // +optional @@ -103,7 +108,7 @@ type TempoStackSpec struct { // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Tempo Component Templates" Template TempoTemplateSpec `json:"template,omitempty"` - // ReplicationFactor is used to define how many component replicas should exist. + // The replication factor is a configuration setting that determines how many ingesters need to acknowledge the data from the distributors before accepting a span. // // +optional // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Replication Factor" @@ -613,6 +618,19 @@ type JaegerQuerySpec struct { // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="ServicesQueryDuration" ServicesQueryDuration *metav1.Duration `json:"servicesQueryDuration,omitempty"` + // FindTracesConcurrentRequests defines how many concurrent request a single trace search can submit (defaults querier.replicas*2). + // The search for traces in Jaeger submits limit+1 requests. First requests finds trace IDs and then it fetches + // entire traces by ID. This property allows Jaeger to fetch traces in parallel. + // Note that by default a single Tempo querier can process 20 concurrent search jobs. + // Increasing this property might require scaling up querier instances, especially on error "job queue full" + // See also Tempo's extraConfig: + // querier.max_concurrent_queries (20 default) + // query_frontend.max_outstanding_per_tenant: (2000 default). Increase if the query-frontend returns 429 + // + // +optional + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="FindTracesConcurrentRequests",xDescriptors="urn:alm:descriptor:com.tectonic.ui:advanced" + FindTracesConcurrentRequests int `json:"findTracesConcurrentRequests,omitempty"` + // Authentication defines the options for the oauth proxy used to protect jaeger UI // // +optional diff --git a/apis/tempo/v1alpha1/zz_generated.deepcopy.go b/apis/tempo/v1alpha1/zz_generated.deepcopy.go index d9bbe161f..f5a9d2534 100644 --- a/apis/tempo/v1alpha1/zz_generated.deepcopy.go +++ b/apis/tempo/v1alpha1/zz_generated.deepcopy.go @@ -1367,6 +1367,7 @@ func (in *TempoMonolithicSpec) DeepCopyInto(out *TempoMonolithicSpec) { *out = new(v1.ResourceRequirements) (*in).DeepCopyInto(*out) } + out.Timeout = in.Timeout if in.ExtraConfig != nil { in, out := &in.ExtraConfig, &out.ExtraConfig *out = new(ExtraConfigSpec) @@ -1508,6 +1509,7 @@ func (in *TempoStackList) DeepCopyObject() runtime.Object { func (in *TempoStackSpec) DeepCopyInto(out *TempoStackSpec) { *out = *in in.LimitSpec.DeepCopyInto(&out.LimitSpec) + out.Timeout = in.Timeout if in.StorageClassName != nil { in, out := &in.StorageClassName, &out.StorageClassName *out = new(string) diff --git a/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml b/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml index d4354f2a6..c8a85e819 100644 --- a/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml +++ b/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml @@ -74,7 +74,7 @@ metadata: capabilities: Deep Insights categories: Logging & Tracing,Monitoring containerImage: ghcr.io/grafana/tempo-operator/tempo-operator:v0.13.0 - createdAt: "2024-10-02T14:56:50Z" + createdAt: "2024-10-14T12:48:35Z" description: Create and manage deployments of Tempo, a high-scale distributed tracing backend. operatorframework.io/cluster-monitoring: "true" @@ -307,6 +307,19 @@ spec: "{"namespace": "", "resource": "pods", "verb": "get"}' displayName: SAR path: jaegerui.authentication.sar + - description: 'FindTracesConcurrentRequests defines how many concurrent request + a single trace search can submit (defaults 2). The search for traces in + Jaeger submits limit+1 requests. First requests finds trace IDs and then + it fetches entire traces by ID. This property allows Jaeger to fetch traces + in parallel. Note that by default a single Tempo querier can process 20 + concurrent search jobs. Increasing this property might require scaling up + querier instances, especially on error "job queue full" See also Tempo''s + extraConfig: querier.max_concurrent_queries (20 default) query_frontend.max_outstanding_per_tenant: + (2000 default). Increase if the query-frontend returns 429' + displayName: FindTracesConcurrentRequests + path: jaegerui.findTracesConcurrentRequests + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced - description: Annotations defines the annotations of the Ingress object. displayName: Annotations path: jaegerui.ingress.annotations @@ -718,8 +731,9 @@ spec: 0 to 1. displayName: Sampling Fraction path: observability.tracing.sampling_fraction - - description: ReplicationFactor is used to define how many component replicas - should exist. + - description: The replication factor is a configuration setting that determines + how many ingesters need to acknowledge the data from the distributors before + accepting a span. displayName: Replication Factor path: replicationFactor - description: Resources defines resources configuration. @@ -1001,6 +1015,19 @@ spec: path: template.queryFrontend.jaegerQuery.enabled x-descriptors: - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - description: 'FindTracesConcurrentRequests defines how many concurrent request + a single trace search can submit (defaults querier.replicas*2). The search + for traces in Jaeger submits limit+1 requests. First requests finds trace + IDs and then it fetches entire traces by ID. This property allows Jaeger + to fetch traces in parallel. Note that by default a single Tempo querier + can process 20 concurrent search jobs. Increasing this property might require + scaling up querier instances, especially on error "job queue full" See also + Tempo''s extraConfig: querier.max_concurrent_queries (20 default) query_frontend.max_outstanding_per_tenant: + (2000 default). Increase if the query-frontend returns 429' + displayName: FindTracesConcurrentRequests + path: template.queryFrontend.jaegerQuery.findTracesConcurrentRequests + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced - description: Ingress defines the options for the Jaeger Query ingress. displayName: Jaeger Query UI Ingress Settings path: template.queryFrontend.jaegerQuery.ingress @@ -1424,15 +1451,15 @@ spec: - name: RELATED_IMAGE_TEMPO value: docker.io/grafana/tempo:2.6.0 - name: RELATED_IMAGE_JAEGER_QUERY - value: docker.io/jaegertracing/jaeger-query:1.60 + value: docker.io/jaegertracing/jaeger-query:1.62.0 - name: RELATED_IMAGE_TEMPO_QUERY - value: docker.io/grafana/tempo-query:main-2999520 + value: docker.io/grafana/tempo-query:main-1de25ca - name: RELATED_IMAGE_TEMPO_GATEWAY value: quay.io/observatorium/api:main-2024-08-05-11d0d94 - name: RELATED_IMAGE_TEMPO_GATEWAY_OPA value: quay.io/observatorium/opa-openshift:main-2024-04-29-914c13f - name: RELATED_IMAGE_OAUTH_PROXY - value: quay.io/openshift/origin-oauth-proxy:4.12 + value: quay.io/openshift/origin-oauth-proxy:4.14 image: ghcr.io/grafana/tempo-operator/tempo-operator:v0.13.0 livenessProbe: httpGet: @@ -1573,15 +1600,15 @@ spec: relatedImages: - image: docker.io/grafana/tempo:2.6.0 name: tempo - - image: docker.io/jaegertracing/jaeger-query:1.60 + - image: docker.io/jaegertracing/jaeger-query:1.62.0 name: jaeger-query - - image: docker.io/grafana/tempo-query:main-2999520 + - image: docker.io/grafana/tempo-query:main-1de25ca name: tempo-query - image: quay.io/observatorium/api:main-2024-08-05-11d0d94 name: tempo-gateway - image: quay.io/observatorium/opa-openshift:main-2024-04-29-914c13f name: tempo-gateway-opa - - image: quay.io/openshift/origin-oauth-proxy:4.12 + - image: quay.io/openshift/origin-oauth-proxy:4.14 name: oauth-proxy version: 0.13.0 webhookdefinitions: diff --git a/bundle/community/manifests/tempo.grafana.com_tempomonolithics.yaml b/bundle/community/manifests/tempo.grafana.com_tempomonolithics.yaml index ae585d652..6d285729c 100644 --- a/bundle/community/manifests/tempo.grafana.com_tempomonolithics.yaml +++ b/bundle/community/manifests/tempo.grafana.com_tempomonolithics.yaml @@ -1109,6 +1109,17 @@ spec: description: Enabled defines if the Jaeger UI component should be created. type: boolean + findTracesConcurrentRequests: + description: |- + FindTracesConcurrentRequests defines how many concurrent request a single trace search can submit (defaults 2). + The search for traces in Jaeger submits limit+1 requests. First requests finds trace IDs and then it fetches + entire traces by ID. This property allows Jaeger to fetch traces in parallel. + Note that by default a single Tempo querier can process 20 concurrent search jobs. + Increasing this property might require scaling up querier instances, especially on error "job queue full" + See also Tempo's extraConfig: + querier.max_concurrent_queries (20 default) + query_frontend.max_outstanding_per_tenant: (2000 default). Increase if the query-frontend returns 429 + type: integer ingress: description: Ingress defines the Ingress configuration for the Jaeger UI. @@ -1683,6 +1694,12 @@ spec: required: - traces type: object + timeout: + description: |- + Timeout configures the same timeout on all components starting at ingress down to the ingestor/querier. + Timeout configuration on a specific component has a higher precedence. + Default is 30 seconds. + type: string tolerations: description: Tolerations defines the tolerations of a node to schedule the pod onto it. diff --git a/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml b/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml index 3d4a67273..6ec9eaffb 100644 --- a/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml +++ b/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml @@ -293,8 +293,9 @@ spec: type: object type: object replicationFactor: - description: ReplicationFactor is used to define how many component - replicas should exist. + description: The replication factor is a configuration setting that + determines how many ingesters need to acknowledge the data from + the distributors before accepting a span. type: integer resources: description: Resources defines resources configuration. @@ -1430,6 +1431,7 @@ spec: enum: - ingress - route + - "" type: string type: object required: @@ -2390,6 +2392,17 @@ spec: description: Enabled defines if the Jaeger Query component should be created. type: boolean + findTracesConcurrentRequests: + description: |- + FindTracesConcurrentRequests defines how many concurrent request a single trace search can submit (defaults querier.replicas*2). + The search for traces in Jaeger submits limit+1 requests. First requests finds trace IDs and then it fetches + entire traces by ID. This property allows Jaeger to fetch traces in parallel. + Note that by default a single Tempo querier can process 20 concurrent search jobs. + Increasing this property might require scaling up querier instances, especially on error "job queue full" + See also Tempo's extraConfig: + querier.max_concurrent_queries (20 default) + query_frontend.max_outstanding_per_tenant: (2000 default). Increase if the query-frontend returns 429 + type: integer ingress: description: Ingress defines the options for the Jaeger Query ingress. @@ -2431,6 +2444,7 @@ spec: enum: - ingress - route + - "" type: string type: object monitorTab: @@ -2714,6 +2728,12 @@ spec: required: - mode type: object + timeout: + description: |- + Timeout configures the same timeout on all components starting at ingress down to the ingestor/querier. + Timeout configuration on a specific component has a higher precedence. + Defaults to 30 seconds. + type: string required: - storage type: object diff --git a/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml b/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml index 90ac637e9..96b3eb990 100644 --- a/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml +++ b/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml @@ -74,7 +74,7 @@ metadata: capabilities: Deep Insights categories: Logging & Tracing,Monitoring containerImage: ghcr.io/grafana/tempo-operator/tempo-operator:v0.13.0 - createdAt: "2024-10-02T14:56:49Z" + createdAt: "2024-10-14T12:48:34Z" description: Create and manage deployments of Tempo, a high-scale distributed tracing backend. operatorframework.io/cluster-monitoring: "true" @@ -307,6 +307,19 @@ spec: "{"namespace": "", "resource": "pods", "verb": "get"}' displayName: SAR path: jaegerui.authentication.sar + - description: 'FindTracesConcurrentRequests defines how many concurrent request + a single trace search can submit (defaults 2). The search for traces in + Jaeger submits limit+1 requests. First requests finds trace IDs and then + it fetches entire traces by ID. This property allows Jaeger to fetch traces + in parallel. Note that by default a single Tempo querier can process 20 + concurrent search jobs. Increasing this property might require scaling up + querier instances, especially on error "job queue full" See also Tempo''s + extraConfig: querier.max_concurrent_queries (20 default) query_frontend.max_outstanding_per_tenant: + (2000 default). Increase if the query-frontend returns 429' + displayName: FindTracesConcurrentRequests + path: jaegerui.findTracesConcurrentRequests + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced - description: Annotations defines the annotations of the Ingress object. displayName: Annotations path: jaegerui.ingress.annotations @@ -718,8 +731,9 @@ spec: 0 to 1. displayName: Sampling Fraction path: observability.tracing.sampling_fraction - - description: ReplicationFactor is used to define how many component replicas - should exist. + - description: The replication factor is a configuration setting that determines + how many ingesters need to acknowledge the data from the distributors before + accepting a span. displayName: Replication Factor path: replicationFactor - description: Resources defines resources configuration. @@ -1001,6 +1015,19 @@ spec: path: template.queryFrontend.jaegerQuery.enabled x-descriptors: - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - description: 'FindTracesConcurrentRequests defines how many concurrent request + a single trace search can submit (defaults querier.replicas*2). The search + for traces in Jaeger submits limit+1 requests. First requests finds trace + IDs and then it fetches entire traces by ID. This property allows Jaeger + to fetch traces in parallel. Note that by default a single Tempo querier + can process 20 concurrent search jobs. Increasing this property might require + scaling up querier instances, especially on error "job queue full" See also + Tempo''s extraConfig: querier.max_concurrent_queries (20 default) query_frontend.max_outstanding_per_tenant: + (2000 default). Increase if the query-frontend returns 429' + displayName: FindTracesConcurrentRequests + path: template.queryFrontend.jaegerQuery.findTracesConcurrentRequests + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced - description: Ingress defines the options for the Jaeger Query ingress. displayName: Jaeger Query UI Ingress Settings path: template.queryFrontend.jaegerQuery.ingress @@ -1434,15 +1461,15 @@ spec: - name: RELATED_IMAGE_TEMPO value: docker.io/grafana/tempo:2.6.0 - name: RELATED_IMAGE_JAEGER_QUERY - value: docker.io/jaegertracing/jaeger-query:1.60 + value: docker.io/jaegertracing/jaeger-query:1.62.0 - name: RELATED_IMAGE_TEMPO_QUERY - value: docker.io/grafana/tempo-query:main-2999520 + value: docker.io/grafana/tempo-query:main-1de25ca - name: RELATED_IMAGE_TEMPO_GATEWAY value: quay.io/observatorium/api:main-2024-08-05-11d0d94 - name: RELATED_IMAGE_TEMPO_GATEWAY_OPA value: quay.io/observatorium/opa-openshift:main-2024-04-29-914c13f - name: RELATED_IMAGE_OAUTH_PROXY - value: quay.io/openshift/origin-oauth-proxy:4.12 + value: quay.io/openshift/origin-oauth-proxy:4.14 image: ghcr.io/grafana/tempo-operator/tempo-operator:v0.13.0 livenessProbe: httpGet: @@ -1594,15 +1621,15 @@ spec: relatedImages: - image: docker.io/grafana/tempo:2.6.0 name: tempo - - image: docker.io/jaegertracing/jaeger-query:1.60 + - image: docker.io/jaegertracing/jaeger-query:1.62.0 name: jaeger-query - - image: docker.io/grafana/tempo-query:main-2999520 + - image: docker.io/grafana/tempo-query:main-1de25ca name: tempo-query - image: quay.io/observatorium/api:main-2024-08-05-11d0d94 name: tempo-gateway - image: quay.io/observatorium/opa-openshift:main-2024-04-29-914c13f name: tempo-gateway-opa - - image: quay.io/openshift/origin-oauth-proxy:4.12 + - image: quay.io/openshift/origin-oauth-proxy:4.14 name: oauth-proxy version: 0.13.0 webhookdefinitions: diff --git a/bundle/openshift/manifests/tempo.grafana.com_tempomonolithics.yaml b/bundle/openshift/manifests/tempo.grafana.com_tempomonolithics.yaml index ae585d652..6d285729c 100644 --- a/bundle/openshift/manifests/tempo.grafana.com_tempomonolithics.yaml +++ b/bundle/openshift/manifests/tempo.grafana.com_tempomonolithics.yaml @@ -1109,6 +1109,17 @@ spec: description: Enabled defines if the Jaeger UI component should be created. type: boolean + findTracesConcurrentRequests: + description: |- + FindTracesConcurrentRequests defines how many concurrent request a single trace search can submit (defaults 2). + The search for traces in Jaeger submits limit+1 requests. First requests finds trace IDs and then it fetches + entire traces by ID. This property allows Jaeger to fetch traces in parallel. + Note that by default a single Tempo querier can process 20 concurrent search jobs. + Increasing this property might require scaling up querier instances, especially on error "job queue full" + See also Tempo's extraConfig: + querier.max_concurrent_queries (20 default) + query_frontend.max_outstanding_per_tenant: (2000 default). Increase if the query-frontend returns 429 + type: integer ingress: description: Ingress defines the Ingress configuration for the Jaeger UI. @@ -1683,6 +1694,12 @@ spec: required: - traces type: object + timeout: + description: |- + Timeout configures the same timeout on all components starting at ingress down to the ingestor/querier. + Timeout configuration on a specific component has a higher precedence. + Default is 30 seconds. + type: string tolerations: description: Tolerations defines the tolerations of a node to schedule the pod onto it. diff --git a/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml b/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml index 3d4a67273..6ec9eaffb 100644 --- a/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml +++ b/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml @@ -293,8 +293,9 @@ spec: type: object type: object replicationFactor: - description: ReplicationFactor is used to define how many component - replicas should exist. + description: The replication factor is a configuration setting that + determines how many ingesters need to acknowledge the data from + the distributors before accepting a span. type: integer resources: description: Resources defines resources configuration. @@ -1430,6 +1431,7 @@ spec: enum: - ingress - route + - "" type: string type: object required: @@ -2390,6 +2392,17 @@ spec: description: Enabled defines if the Jaeger Query component should be created. type: boolean + findTracesConcurrentRequests: + description: |- + FindTracesConcurrentRequests defines how many concurrent request a single trace search can submit (defaults querier.replicas*2). + The search for traces in Jaeger submits limit+1 requests. First requests finds trace IDs and then it fetches + entire traces by ID. This property allows Jaeger to fetch traces in parallel. + Note that by default a single Tempo querier can process 20 concurrent search jobs. + Increasing this property might require scaling up querier instances, especially on error "job queue full" + See also Tempo's extraConfig: + querier.max_concurrent_queries (20 default) + query_frontend.max_outstanding_per_tenant: (2000 default). Increase if the query-frontend returns 429 + type: integer ingress: description: Ingress defines the options for the Jaeger Query ingress. @@ -2431,6 +2444,7 @@ spec: enum: - ingress - route + - "" type: string type: object monitorTab: @@ -2714,6 +2728,12 @@ spec: required: - mode type: object + timeout: + description: |- + Timeout configures the same timeout on all components starting at ingress down to the ingestor/querier. + Timeout configuration on a specific component has a higher precedence. + Defaults to 30 seconds. + type: string required: - storage type: object diff --git a/config/crd/bases/tempo.grafana.com_tempomonolithics.yaml b/config/crd/bases/tempo.grafana.com_tempomonolithics.yaml index 4d28c00ea..918f93e42 100644 --- a/config/crd/bases/tempo.grafana.com_tempomonolithics.yaml +++ b/config/crd/bases/tempo.grafana.com_tempomonolithics.yaml @@ -1105,6 +1105,17 @@ spec: description: Enabled defines if the Jaeger UI component should be created. type: boolean + findTracesConcurrentRequests: + description: |- + FindTracesConcurrentRequests defines how many concurrent request a single trace search can submit (defaults 2). + The search for traces in Jaeger submits limit+1 requests. First requests finds trace IDs and then it fetches + entire traces by ID. This property allows Jaeger to fetch traces in parallel. + Note that by default a single Tempo querier can process 20 concurrent search jobs. + Increasing this property might require scaling up querier instances, especially on error "job queue full" + See also Tempo's extraConfig: + querier.max_concurrent_queries (20 default) + query_frontend.max_outstanding_per_tenant: (2000 default). Increase if the query-frontend returns 429 + type: integer ingress: description: Ingress defines the Ingress configuration for the Jaeger UI. @@ -1679,6 +1690,12 @@ spec: required: - traces type: object + timeout: + description: |- + Timeout configures the same timeout on all components starting at ingress down to the ingestor/querier. + Timeout configuration on a specific component has a higher precedence. + Default is 30 seconds. + type: string tolerations: description: Tolerations defines the tolerations of a node to schedule the pod onto it. diff --git a/config/crd/bases/tempo.grafana.com_tempostacks.yaml b/config/crd/bases/tempo.grafana.com_tempostacks.yaml index 264b9cd3e..9b156e754 100644 --- a/config/crd/bases/tempo.grafana.com_tempostacks.yaml +++ b/config/crd/bases/tempo.grafana.com_tempostacks.yaml @@ -289,8 +289,9 @@ spec: type: object type: object replicationFactor: - description: ReplicationFactor is used to define how many component - replicas should exist. + description: The replication factor is a configuration setting that + determines how many ingesters need to acknowledge the data from + the distributors before accepting a span. type: integer resources: description: Resources defines resources configuration. @@ -1426,6 +1427,7 @@ spec: enum: - ingress - route + - "" type: string type: object required: @@ -2386,6 +2388,17 @@ spec: description: Enabled defines if the Jaeger Query component should be created. type: boolean + findTracesConcurrentRequests: + description: |- + FindTracesConcurrentRequests defines how many concurrent request a single trace search can submit (defaults querier.replicas*2). + The search for traces in Jaeger submits limit+1 requests. First requests finds trace IDs and then it fetches + entire traces by ID. This property allows Jaeger to fetch traces in parallel. + Note that by default a single Tempo querier can process 20 concurrent search jobs. + Increasing this property might require scaling up querier instances, especially on error "job queue full" + See also Tempo's extraConfig: + querier.max_concurrent_queries (20 default) + query_frontend.max_outstanding_per_tenant: (2000 default). Increase if the query-frontend returns 429 + type: integer ingress: description: Ingress defines the options for the Jaeger Query ingress. @@ -2427,6 +2440,7 @@ spec: enum: - ingress - route + - "" type: string type: object monitorTab: @@ -2710,6 +2724,12 @@ spec: required: - mode type: object + timeout: + description: |- + Timeout configures the same timeout on all components starting at ingress down to the ingestor/querier. + Timeout configuration on a specific component has a higher precedence. + Defaults to 30 seconds. + type: string required: - storage type: object diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 392a5cd98..a7822e56a 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -42,15 +42,15 @@ spec: - name: RELATED_IMAGE_TEMPO value: docker.io/grafana/tempo:2.6.0 - name: RELATED_IMAGE_JAEGER_QUERY - value: docker.io/jaegertracing/jaeger-query:1.60 + value: docker.io/jaegertracing/jaeger-query:1.62.0 - name: RELATED_IMAGE_TEMPO_QUERY - value: docker.io/grafana/tempo-query:main-2999520 + value: docker.io/grafana/tempo-query:main-1de25ca - name: RELATED_IMAGE_TEMPO_GATEWAY value: quay.io/observatorium/api:main-2024-08-05-11d0d94 - name: RELATED_IMAGE_TEMPO_GATEWAY_OPA value: quay.io/observatorium/opa-openshift:main-2024-04-29-914c13f - name: RELATED_IMAGE_OAUTH_PROXY - value: quay.io/openshift/origin-oauth-proxy:4.12 + value: quay.io/openshift/origin-oauth-proxy:4.14 securityContext: allowPrivilegeEscalation: false capabilities: diff --git a/config/manifests/community/bases/tempo-operator.clusterserviceversion.yaml b/config/manifests/community/bases/tempo-operator.clusterserviceversion.yaml index 3d85811b5..183b5d776 100644 --- a/config/manifests/community/bases/tempo-operator.clusterserviceversion.yaml +++ b/config/manifests/community/bases/tempo-operator.clusterserviceversion.yaml @@ -236,6 +236,19 @@ spec: "{"namespace": "", "resource": "pods", "verb": "get"}' displayName: SAR path: jaegerui.authentication.sar + - description: 'FindTracesConcurrentRequests defines how many concurrent request + a single trace search can submit (defaults 2). The search for traces in + Jaeger submits limit+1 requests. First requests finds trace IDs and then + it fetches entire traces by ID. This property allows Jaeger to fetch traces + in parallel. Note that by default a single Tempo querier can process 20 + concurrent search jobs. Increasing this property might require scaling up + querier instances, especially on error "job queue full" See also Tempo''s + extraConfig: querier.max_concurrent_queries (20 default) query_frontend.max_outstanding_per_tenant: + (2000 default). Increase if the query-frontend returns 429' + displayName: FindTracesConcurrentRequests + path: jaegerui.findTracesConcurrentRequests + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced - description: Annotations defines the annotations of the Ingress object. displayName: Annotations path: jaegerui.ingress.annotations @@ -647,8 +660,9 @@ spec: 0 to 1. displayName: Sampling Fraction path: observability.tracing.sampling_fraction - - description: ReplicationFactor is used to define how many component replicas - should exist. + - description: The replication factor is a configuration setting that determines + how many ingesters need to acknowledge the data from the distributors before + accepting a span. displayName: Replication Factor path: replicationFactor - description: Resources defines resources configuration. @@ -930,6 +944,19 @@ spec: path: template.queryFrontend.jaegerQuery.enabled x-descriptors: - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - description: 'FindTracesConcurrentRequests defines how many concurrent request + a single trace search can submit (defaults querier.replicas*2). The search + for traces in Jaeger submits limit+1 requests. First requests finds trace + IDs and then it fetches entire traces by ID. This property allows Jaeger + to fetch traces in parallel. Note that by default a single Tempo querier + can process 20 concurrent search jobs. Increasing this property might require + scaling up querier instances, especially on error "job queue full" See also + Tempo''s extraConfig: querier.max_concurrent_queries (20 default) query_frontend.max_outstanding_per_tenant: + (2000 default). Increase if the query-frontend returns 429' + displayName: FindTracesConcurrentRequests + path: template.queryFrontend.jaegerQuery.findTracesConcurrentRequests + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced - description: Ingress defines the options for the Jaeger Query ingress. displayName: Jaeger Query UI Ingress Settings path: template.queryFrontend.jaegerQuery.ingress diff --git a/config/manifests/openshift/bases/tempo-operator.clusterserviceversion.yaml b/config/manifests/openshift/bases/tempo-operator.clusterserviceversion.yaml index 9a1fda20f..55a0cd54a 100644 --- a/config/manifests/openshift/bases/tempo-operator.clusterserviceversion.yaml +++ b/config/manifests/openshift/bases/tempo-operator.clusterserviceversion.yaml @@ -236,6 +236,19 @@ spec: "{"namespace": "", "resource": "pods", "verb": "get"}' displayName: SAR path: jaegerui.authentication.sar + - description: 'FindTracesConcurrentRequests defines how many concurrent request + a single trace search can submit (defaults 2). The search for traces in + Jaeger submits limit+1 requests. First requests finds trace IDs and then + it fetches entire traces by ID. This property allows Jaeger to fetch traces + in parallel. Note that by default a single Tempo querier can process 20 + concurrent search jobs. Increasing this property might require scaling up + querier instances, especially on error "job queue full" See also Tempo''s + extraConfig: querier.max_concurrent_queries (20 default) query_frontend.max_outstanding_per_tenant: + (2000 default). Increase if the query-frontend returns 429' + displayName: FindTracesConcurrentRequests + path: jaegerui.findTracesConcurrentRequests + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced - description: Annotations defines the annotations of the Ingress object. displayName: Annotations path: jaegerui.ingress.annotations @@ -647,8 +660,9 @@ spec: 0 to 1. displayName: Sampling Fraction path: observability.tracing.sampling_fraction - - description: ReplicationFactor is used to define how many component replicas - should exist. + - description: The replication factor is a configuration setting that determines + how many ingesters need to acknowledge the data from the distributors before + accepting a span. displayName: Replication Factor path: replicationFactor - description: Resources defines resources configuration. @@ -930,6 +944,19 @@ spec: path: template.queryFrontend.jaegerQuery.enabled x-descriptors: - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - description: 'FindTracesConcurrentRequests defines how many concurrent request + a single trace search can submit (defaults querier.replicas*2). The search + for traces in Jaeger submits limit+1 requests. First requests finds trace + IDs and then it fetches entire traces by ID. This property allows Jaeger + to fetch traces in parallel. Note that by default a single Tempo querier + can process 20 concurrent search jobs. Increasing this property might require + scaling up querier instances, especially on error "job queue full" See also + Tempo''s extraConfig: querier.max_concurrent_queries (20 default) query_frontend.max_outstanding_per_tenant: + (2000 default). Increase if the query-frontend returns 429' + displayName: FindTracesConcurrentRequests + path: template.queryFrontend.jaegerQuery.findTracesConcurrentRequests + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced - description: Ingress defines the options for the Jaeger Query ingress. displayName: Jaeger Query UI Ingress Settings path: template.queryFrontend.jaegerQuery.ingress diff --git a/docs/spec/tempo.grafana.com_tempomonolithics.yaml b/docs/spec/tempo.grafana.com_tempomonolithics.yaml index 3ab1b69a4..f4ab9605b 100644 --- a/docs/spec/tempo.grafana.com_tempomonolithics.yaml +++ b/docs/spec/tempo.grafana.com_tempomonolithics.yaml @@ -35,6 +35,7 @@ spec: # TempoMonolithicSpec defines the desir requests: # Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. Requests cannot exceed Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ cpu: "500m" memory: "1Gi" + findTracesConcurrentRequests: 0 # FindTracesConcurrentRequests defines how many concurrent request a single trace search can submit (defaults 2). The search for traces in Jaeger submits limit+1 requests. First requests finds trace IDs and then it fetches entire traces by ID. This property allows Jaeger to fetch traces in parallel. Note that by default a single Tempo querier can process 20 concurrent search jobs. Increasing this property might require scaling up querier instances, especially on error "job queue full" See also Tempo's extraConfig: querier.max_concurrent_queries (20 default) query_frontend.max_outstanding_per_tenant: (2000 default). Increase if the query-frontend returns 429 ingress: # Ingress defines the Ingress configuration for the Jaeger UI. enabled: false # Enabled defines if an Ingress object should be created for Jaeger UI. annotations: {} # Annotations defines the annotations of the Ingress object. @@ -126,6 +127,7 @@ spec: # TempoMonolithicSpec defines the desir certName: "" # Cert is the name of a Secret containing a certificate (tls.crt) and private key (tls.key). It needs to be in the same namespace as the Tempo custom resource. minVersion: "" # MinVersion defines the minimum acceptable TLS version. size: 0Gi # Size defines the size of the volume where traces are stored. For in-memory storage, this defines the size of the tmpfs volume. For persistent volume storage, this defines the size of the persistent volume. For object storage, this defines the size of the persistent volume containing the Write-Ahead Log (WAL) of Tempo. Default: 2Gi for memory, 10Gi for all other backends. + timeout: "" # Timeout configures the same timeout on all components starting at ingress down to the ingestor/querier. Timeout configuration on a specific component has a higher precedence. Default is 30 seconds. affinity: # Affinity defines the Affinity rules for scheduling pods. nodeAffinity: {} # Describes node affinity scheduling rules for the pod. podAffinity: {} # Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). diff --git a/docs/spec/tempo.grafana.com_tempostacks.yaml b/docs/spec/tempo.grafana.com_tempostacks.yaml index 3693381ad..ac32bdb41 100644 --- a/docs/spec/tempo.grafana.com_tempostacks.yaml +++ b/docs/spec/tempo.grafana.com_tempostacks.yaml @@ -54,7 +54,7 @@ spec: # TempoStackSpec defines the desired st tracing: # Tracing defines a config for operands. jaeger_agent_endpoint: "localhost:6831" # JaegerAgentEndpoint defines the jaeger endpoint data gets send to. sampling_fraction: "" # SamplingFraction defines the sampling ratio. Valid values are 0 to 1. - replicationFactor: 0 # ReplicationFactor is used to define how many component replicas should exist. + replicationFactor: 0 # The replication factor is a configuration setting that determines how many ingesters need to acknowledge the data from the distributors before accepting a span. retention: # Retention period defined by dataset. User can specify how long data should be stored. global: # Global is used to configure global retention. traces: "" # Traces defines retention period. Supported parameter suffixes are "s", "m" and "h". example: 336h default: value is 48h. @@ -330,6 +330,7 @@ spec: # TempoStackSpec defines the desired st requests: # Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. Requests cannot exceed Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ cpu: "500m" memory: "1Gi" + findTracesConcurrentRequests: 0 # FindTracesConcurrentRequests defines how many concurrent request a single trace search can submit (defaults querier.replicas*2). The search for traces in Jaeger submits limit+1 requests. First requests finds trace IDs and then it fetches entire traces by ID. This property allows Jaeger to fetch traces in parallel. Note that by default a single Tempo querier can process 20 concurrent search jobs. Increasing this property might require scaling up querier instances, especially on error "job queue full" See also Tempo's extraConfig: querier.max_concurrent_queries (20 default) query_frontend.max_outstanding_per_tenant: (2000 default). Increase if the query-frontend returns 429 ingress: # Ingress defines the options for the Jaeger Query ingress. annotations: {} # Annotations defines the annotations of the Ingress object. host: "" # Host defines the hostname of the Ingress object. @@ -388,6 +389,7 @@ spec: # TempoStackSpec defines the desired st resources: - "" mode: "static" # Mode defines the multitenancy mode. + timeout: "" # Timeout configures the same timeout on all components starting at ingress down to the ingestor/querier. Timeout configuration on a specific component has a higher precedence. Defaults to 30 seconds. resources: # Resources defines resources configuration. total: # The total amount of resources for Tempo instance. The operator autonomously splits resources between deployed Tempo components. Only limits are supported, the operator calculates requests automatically. See http://github.com/grafana/tempo/issues/1540. claims: # Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. This is an alpha field and requires enabling the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. diff --git a/internal/manifests/config/build.go b/internal/manifests/config/build.go index 489a2c9fe..9b17bc705 100644 --- a/internal/manifests/config/build.go +++ b/internal/manifests/config/build.go @@ -89,6 +89,7 @@ func buildConfiguration(params manifestutils.Params) ([]byte, error) { TLS: tlsopts, ReceiverTLS: buildReceiverTLSConfig(tempo), S3StorageTLS: buildS3StorageTLSConfig(params), + Timeout: params.Tempo.Spec.Timeout.Duration, } if isTenantOverridesConfigRequired(tempo.Spec.LimitSpec) { @@ -168,6 +169,14 @@ func buildTempoQueryConfig(params manifestutils.Params) ([]byte, error) { return []byte{}, err } + findTracesConcurrentRequests := params.Tempo.Spec.Template.QueryFrontend.JaegerQuery.FindTracesConcurrentRequests + if findTracesConcurrentRequests == 0 { + querierReplicas := int32(1) + if params.Tempo.Spec.Template.Querier.Replicas != nil { + querierReplicas = *params.Tempo.Spec.Template.Querier.Replicas + } + findTracesConcurrentRequests = int(querierReplicas) * 2 + } return renderTempoQueryTemplate(tempoQueryOptions{ TLS: tlsopts, HTTPPort: manifestutils.PortHTTPServer, @@ -175,9 +184,10 @@ func buildTempoQueryConfig(params manifestutils.Params) ([]byte, error) { GRPCEncryption: params.CtrlConfig.Gates.GRPCEncryption, HTTPEncryption: params.CtrlConfig.Gates.HTTPEncryption, }, - TenantHeader: manifestutils.TenantHeader, - Gateway: params.Tempo.Spec.Template.Gateway.Enabled, - ServicesQueryDuration: params.Tempo.Spec.Template.QueryFrontend.JaegerQuery.ServicesQueryDuration.Duration.String(), + TenantHeader: manifestutils.TenantHeader, + Gateway: params.Tempo.Spec.Template.Gateway.Enabled, + ServicesQueryDuration: params.Tempo.Spec.Template.QueryFrontend.JaegerQuery.ServicesQueryDuration.Duration.String(), + FindTracesConcurrentRequests: findTracesConcurrentRequests, }) } diff --git a/internal/manifests/config/build_test.go b/internal/manifests/config/build_test.go index 62c50571e..23623d48d 100644 --- a/internal/manifests/config/build_test.go +++ b/internal/manifests/config/build_test.go @@ -74,8 +74,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 30s + http_server_write_timeout: 30s log_format: logfmt storage: trace: @@ -103,6 +103,7 @@ query_frontend: Name: "test", }, Spec: v1alpha1.TempoStackSpec{ + Timeout: metav1.Duration{Duration: time.Second * 30}, Storage: v1alpha1.ObjectStorageSpec{ Secret: v1alpha1.ObjectStorageSecretSpec{ Type: v1alpha1.ObjectStorageSecretS3, @@ -197,8 +198,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -286,8 +287,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -374,8 +375,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -463,8 +464,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -552,8 +553,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -641,8 +642,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -730,8 +731,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -831,8 +832,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -930,8 +931,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -964,6 +965,7 @@ query_frontend: Name: "test", }, Spec: v1alpha1.TempoStackSpec{ + Timeout: metav1.Duration{Duration: time.Minute * 3}, Storage: v1alpha1.ObjectStorageSpec{ Secret: v1alpha1.ObjectStorageSecretSpec{ Type: v1alpha1.ObjectStorageSecretS3, @@ -1091,8 +1093,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -1124,6 +1126,7 @@ query_frontend: Name: "test", }, Spec: v1alpha1.TempoStackSpec{ + Timeout: metav1.Duration{Duration: time.Minute * 3}, Storage: v1alpha1.ObjectStorageSpec{ Secret: v1alpha1.ObjectStorageSecretSpec{ Type: v1alpha1.ObjectStorageSecretGCS, @@ -1205,8 +1208,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -1233,6 +1236,7 @@ query_frontend: Name: "test", }, Spec: v1alpha1.TempoStackSpec{ + Timeout: metav1.Duration{Duration: time.Minute * 3}, Storage: v1alpha1.ObjectStorageSpec{ Secret: v1alpha1.ObjectStorageSecretSpec{ Type: v1alpha1.ObjectStorageSecretAzure, @@ -1309,8 +1313,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -1338,6 +1342,7 @@ query_frontend: Name: "test", }, Spec: v1alpha1.TempoStackSpec{ + Timeout: metav1.Duration{Duration: time.Minute * 3}, Storage: v1alpha1.ObjectStorageSpec{ Secret: v1alpha1.ObjectStorageSecretSpec{ Type: v1alpha1.ObjectStorageSecretS3, @@ -1455,8 +1460,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt tls_cipher_suites: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 tls_min_version: VersionTLS12 @@ -1574,8 +1579,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt tls_min_version: VersionTLS13 grpc_tls_config: @@ -1630,6 +1635,7 @@ ingester_client: Namespace: "nstest", }, Spec: v1alpha1.TempoStackSpec{ + Timeout: metav1.Duration{Duration: time.Minute * 3}, Storage: v1alpha1.ObjectStorageSpec{ Secret: v1alpha1.ObjectStorageSecretSpec{ Type: v1alpha1.ObjectStorageSecretS3, @@ -1737,8 +1743,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt tls_cipher_suites: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 tls_min_version: VersionTLS12 @@ -1789,6 +1795,7 @@ ingester_client: Namespace: "nstest", }, Spec: v1alpha1.TempoStackSpec{ + Timeout: metav1.Duration{Duration: 3 * time.Minute}, Template: v1alpha1.TempoTemplateSpec{ Gateway: v1alpha1.TempoGatewaySpec{ Enabled: true, @@ -1902,8 +1909,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -2010,8 +2017,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -2124,8 +2131,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -2158,6 +2165,7 @@ query_frontend: Name: "test", }, Spec: v1alpha1.TempoStackSpec{ + Timeout: metav1.Duration{Duration: time.Minute * 3}, Storage: v1alpha1.ObjectStorageSpec{ Secret: v1alpha1.ObjectStorageSecretSpec{ Type: v1alpha1.ObjectStorageSecretS3, @@ -2257,8 +2265,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -2340,8 +2348,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -2374,6 +2382,7 @@ query_frontend: Name: "test", }, Spec: v1alpha1.TempoStackSpec{ + Timeout: metav1.Duration{Duration: 3 * time.Minute}, Storage: v1alpha1.ObjectStorageSpec{ Secret: v1alpha1.ObjectStorageSecretSpec{ Type: v1alpha1.ObjectStorageSecretS3, @@ -2464,8 +2473,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -2492,6 +2501,7 @@ query_frontend: Name: "test", }, Spec: v1alpha1.TempoStackSpec{ + Timeout: metav1.Duration{Duration: time.Minute * 3}, Storage: v1alpha1.ObjectStorageSpec{ Secret: v1alpha1.ObjectStorageSecretSpec{ Type: v1alpha1.ObjectStorageSecretS3, diff --git a/internal/manifests/config/options.go b/internal/manifests/config/options.go index c3a220b3b..d34519764 100644 --- a/internal/manifests/config/options.go +++ b/internal/manifests/config/options.go @@ -1,6 +1,10 @@ package config -import "github.com/grafana/tempo-operator/internal/manifests/manifestutils" +import ( + "time" + + "github.com/grafana/tempo-operator/internal/manifests/manifestutils" +) // options holds the configuration template options. type options struct { @@ -19,15 +23,17 @@ type options struct { Gates featureGates ReceiverTLS receiverTLSOptions S3StorageTLS storageTLSOptions + Timeout time.Duration } type tempoQueryOptions struct { - Gates featureGates - TLS tlsOptions - HTTPPort int - TenantHeader string - Gateway bool - ServicesQueryDuration string + Gates featureGates + TLS tlsOptions + HTTPPort int + TenantHeader string + Gateway bool + ServicesQueryDuration string + FindTracesConcurrentRequests int } type featureGates struct { diff --git a/internal/manifests/config/tempo-config.yaml b/internal/manifests/config/tempo-config.yaml index f43141207..cee5abb49 100644 --- a/internal/manifests/config/tempo-config.yaml +++ b/internal/manifests/config/tempo-config.yaml @@ -182,8 +182,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: {{ .Timeout }} + http_server_write_timeout: {{ .Timeout }} log_format: logfmt {{- if or .Gates.GRPCEncryption .Gates.HTTPEncryption }} {{- if .TLS.Profile.Ciphers }} diff --git a/internal/manifests/config/tempo-query.yaml b/internal/manifests/config/tempo-query.yaml index 4096bc107..782c95d3d 100644 --- a/internal/manifests/config/tempo-query.yaml +++ b/internal/manifests/config/tempo-query.yaml @@ -10,3 +10,4 @@ tls_insecure_skip_verify: false tls_server_name: {{ .TLS.ServerNames.QueryFrontend }} {{- end }} services_query_duration: {{ .ServicesQueryDuration }} +find_traces_concurrent_requests: {{ .FindTracesConcurrentRequests }} diff --git a/internal/manifests/gateway/gateway.go b/internal/manifests/gateway/gateway.go index 5b778475d..aff673e8a 100644 --- a/internal/manifests/gateway/gateway.go +++ b/internal/manifests/gateway/gateway.go @@ -248,6 +248,7 @@ func deployment(params manifestutils.Params, rbacCfgHash string, tenantsCfgHash fmt.Sprintf("--web.internal.listen=0.0.0.0:%d", manifestutils.GatewayPortInternalHTTPServer), // serves health checks fmt.Sprintf("--traces.write.otlpgrpc.endpoint=%s:%d", naming.ServiceFqdn(tempo.Namespace, tempo.Name, manifestutils.DistributorComponentName), manifestutils.PortOtlpGrpcServer), // Tempo Distributor gRPC upstream fmt.Sprintf("--traces.write.otlphttp.endpoint=%s://%s:%d", httpScheme(params.CtrlConfig.Gates.HTTPEncryption), naming.ServiceFqdn(tempo.Namespace, tempo.Name, manifestutils.DistributorComponentName), manifestutils.PortOtlpHttp), // Tempo Distributor HTTP upstream + fmt.Sprintf("--traces.write-timeout=%s", params.Tempo.Spec.Timeout.Duration.String()), fmt.Sprintf("--traces.tempo.endpoint=%s://%s:%d", httpScheme(params.CtrlConfig.Gates.HTTPEncryption), naming.ServiceFqdn(tempo.Namespace, tempo.Name, manifestutils.QueryFrontendComponentName), manifestutils.PortHTTPServer), // Tempo API upstream fmt.Sprintf("--grpc.listen=0.0.0.0:%d", manifestutils.GatewayPortGRPCServer), // proxies Tempo Distributor gRPC diff --git a/internal/manifests/gateway/gateway_test.go b/internal/manifests/gateway/gateway_test.go index 5439d07a7..3feca416b 100644 --- a/internal/manifests/gateway/gateway_test.go +++ b/internal/manifests/gateway/gateway_test.go @@ -2,7 +2,6 @@ package gateway import ( "fmt" - "net" "reflect" "testing" @@ -249,7 +248,7 @@ func TestBuildGateway_openshift(t *testing.T) { require.True(t, ok) require.Equal(t, "Service", route.Spec.To.Kind) require.Equal(t, "tempo-simplest-gateway", route.Spec.To.Name) - require.Equal(t, map[string]string{"timeout": "30s"}, route.ObjectMeta.Annotations) + require.Equal(t, map[string]string{"timeout": "30s", "haproxy.router.openshift.io/timeout": "0s"}, route.ObjectMeta.Annotations) obj = getObjectByTypeAndName(objects, "tempo-simplest-gateway-cabundle", reflect.TypeOf(&corev1.ConfigMap{})) require.NotNil(t, obj) @@ -775,6 +774,9 @@ func TestRoute(t *testing.T) { Name: naming.Name(manifestutils.GatewayComponentName, "test"), Namespace: "project1", Labels: manifestutils.ComponentLabels("gateway", "test"), + Annotations: map[string]string{ + "haproxy.router.openshift.io/timeout": "0s", + }, }, Spec: routev1.RouteSpec{ To: routev1.RouteTargetReference{ diff --git a/internal/manifests/gateway/openshift.go b/internal/manifests/gateway/openshift.go index 858564042..7878db200 100644 --- a/internal/manifests/gateway/openshift.go +++ b/internal/manifests/gateway/openshift.go @@ -22,6 +22,8 @@ import ( const ( gatewayOPAHTTPPort = 8082 gatewayOPAInternalPort = 8083 + + timeoutRouteAnnotation = "haproxy.router.openshift.io/timeout" ) // BuildServiceAccountAnnotations returns the annotations to use a ServiceAccount as an OAuth client. @@ -111,12 +113,20 @@ func route(tempo v1alpha1.TempoStack) (*routev1.Route, error) { return nil, fmt.Errorf("unsupported tls termination specified for route") } + annotations := tempo.Spec.Template.Gateway.Ingress.Annotations + if annotations == nil { + annotations = map[string]string{} + } + if annotations[timeoutRouteAnnotation] == "" { + annotations[timeoutRouteAnnotation] = fmt.Sprintf("%ds", int(tempo.Spec.Timeout.Duration.Seconds())) + } + return &routev1.Route{ ObjectMeta: metav1.ObjectMeta{ Name: naming.Name(manifestutils.GatewayComponentName, tempo.Name), Namespace: tempo.Namespace, Labels: labels, - Annotations: tempo.Spec.Template.Gateway.Ingress.Annotations, + Annotations: annotations, }, Spec: routev1.RouteSpec{ Host: tempo.Spec.Template.Gateway.Ingress.Host, diff --git a/internal/manifests/manifests_test.go b/internal/manifests/manifests_test.go index a76404b6f..dfeeddeff 100644 --- a/internal/manifests/manifests_test.go +++ b/internal/manifests/manifests_test.go @@ -2,6 +2,7 @@ package manifests import ( "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -33,6 +34,7 @@ func TestBuildAll(t *testing.T) { Namespace: "project1", }, Spec: v1alpha1.TempoStackSpec{ + Timeout: metav1.Duration{Duration: time.Second * 5}, Template: v1alpha1.TempoTemplateSpec{ Gateway: v1alpha1.TempoGatewaySpec{ Enabled: true, diff --git a/internal/manifests/monolithic/build.go b/internal/manifests/monolithic/build.go index b084eeed4..3c16670ce 100644 --- a/internal/manifests/monolithic/build.go +++ b/internal/manifests/monolithic/build.go @@ -96,6 +96,7 @@ func BuildAll(opts Options) ([]client.Object, error) { oauthproxy.PatchStatefulSetForOauthProxy( tempo.ObjectMeta, tempo.Spec.JaegerUI.Authentication, + tempo.Spec.Timeout.Duration, opts.CtrlConfig, statefulSet) oauthproxy.PatchQueryFrontEndService(getJaegerUIService(services, tempo), tempo.Name) diff --git a/internal/manifests/monolithic/configmap.go b/internal/manifests/monolithic/configmap.go index 967414f61..c9ae7ba2f 100644 --- a/internal/manifests/monolithic/configmap.go +++ b/internal/manifests/monolithic/configmap.go @@ -54,9 +54,11 @@ type tempoConfig struct { MultitenancyEnabled bool `yaml:"multitenancy_enabled,omitempty"` Server struct { - HTTPListenAddress string `yaml:"http_listen_address,omitempty"` - HttpListenPort int `yaml:"http_listen_port,omitempty"` - GRPCListenAddress string `yaml:"grpc_listen_address,omitempty"` + HTTPListenAddress string `yaml:"http_listen_address,omitempty"` + HttpListenPort int `yaml:"http_listen_port,omitempty"` + GRPCListenAddress string `yaml:"grpc_listen_address,omitempty"` + HttpServerReadTimeout time.Duration `yaml:"http_server_read_timeout,omitempty"` + HttpServerWriteTimeout time.Duration `yaml:"http_server_write_timeout,omitempty"` } `yaml:"server"` InternalServer struct { @@ -94,10 +96,11 @@ type tempoConfig struct { } type tempoQueryConfig struct { - Address string `yaml:"address"` - Backend string `yaml:"backend"` - TenantHeaderKey string `yaml:"tenant_header_key"` - ServicesQueryDuration time.Duration `yaml:"services_query_duration"` + Address string `yaml:"address"` + Backend string `yaml:"backend"` + TenantHeaderKey string `yaml:"tenant_header_key"` + ServicesQueryDuration time.Duration `yaml:"services_query_duration"` + FindTracesConcurrentRequests int `yaml:"find_traces_concurrent_requests"` } // BuildConfigMap creates the Tempo ConfigMap for a monolithic deployment. @@ -170,6 +173,8 @@ func buildTempoConfig(opts Options) ([]byte, error) { config := tempoConfig{} config.MultitenancyEnabled = tempo.Spec.Multitenancy != nil && tempo.Spec.Multitenancy.Enabled config.Server.HttpListenPort = manifestutils.PortHTTPServer + config.Server.HttpServerReadTimeout = opts.Tempo.Spec.Timeout.Duration + config.Server.HttpServerWriteTimeout = opts.Tempo.Spec.Timeout.Duration if tempo.Spec.Multitenancy.IsGatewayEnabled() { // all connections to tempo must go via gateway config.Server.HTTPListenAddress = "localhost" @@ -290,5 +295,6 @@ func buildTempoQueryConfig(jaegerUISpec *v1alpha1.MonolithicJaegerUISpec) ([]byt config.Backend = fmt.Sprintf("127.0.0.1:%d", manifestutils.PortHTTPServer) config.TenantHeaderKey = manifestutils.TenantHeader config.ServicesQueryDuration = jaegerUISpec.ServicesQueryDuration.Duration + config.FindTracesConcurrentRequests = jaegerUISpec.FindTracesConcurrentRequests return yaml.Marshal(&config) } diff --git a/internal/manifests/monolithic/configmap_test.go b/internal/manifests/monolithic/configmap_test.go index f7f31270f..79ed15f45 100644 --- a/internal/manifests/monolithic/configmap_test.go +++ b/internal/manifests/monolithic/configmap_test.go @@ -42,8 +42,9 @@ func TestBuildConfigMap(t *testing.T) { }, }, JaegerUI: &v1alpha1.MonolithicJaegerUISpec{ - Enabled: true, - ServicesQueryDuration: &metav1.Duration{Duration: time.Duration(3 * 24 * time.Hour)}, + Enabled: true, + ServicesQueryDuration: &metav1.Duration{Duration: time.Duration(3 * 24 * time.Hour)}, + FindTracesConcurrentRequests: 22, }, }, }, @@ -63,6 +64,7 @@ address: 127.0.0.1:7777 backend: 127.0.0.1:3200 tenant_header_key: x-scope-orgid services_query_duration: 72h0m0s +find_traces_concurrent_requests: 22 ` require.YAMLEq(t, tempoQueryCfg, cm.Data["tempo-query.yaml"]) } @@ -80,6 +82,8 @@ func TestBuildConfig(t *testing.T) { expected: ` server: http_listen_port: 3200 + http_server_read_timeout: 30s + http_server_write_timeout: 30s internal_server: enable: true http_listen_address: 0.0.0.0 @@ -112,6 +116,8 @@ usage_report: expected: ` server: http_listen_port: 3200 + http_server_read_timeout: 30s + http_server_write_timeout: 30s internal_server: enable: true http_listen_address: 0.0.0.0 @@ -155,6 +161,8 @@ usage_report: expected: ` server: http_listen_port: 3200 + http_server_read_timeout: 30s + http_server_write_timeout: 30s internal_server: enable: true http_listen_address: 0.0.0.0 @@ -202,6 +210,8 @@ usage_report: expected: ` server: http_listen_port: 3200 + http_server_read_timeout: 30s + http_server_write_timeout: 30s internal_server: enable: true http_listen_address: 0.0.0.0 @@ -255,6 +265,8 @@ usage_report: expected: ` server: http_listen_port: 3200 + http_server_read_timeout: 30s + http_server_write_timeout: 30s internal_server: enable: true http_listen_address: 0.0.0.0 @@ -313,6 +325,8 @@ usage_report: expected: ` server: http_listen_port: 3200 + http_server_read_timeout: 30s + http_server_write_timeout: 30s internal_server: enable: true http_listen_address: 0.0.0.0 @@ -369,6 +383,8 @@ usage_report: expected: ` server: http_listen_port: 3200 + http_server_read_timeout: 30s + http_server_write_timeout: 30s internal_server: enable: true http_listen_address: 0.0.0.0 @@ -397,12 +413,14 @@ usage_report: name: "extra config", spec: v1alpha1.TempoMonolithicSpec{ ExtraConfig: &v1alpha1.ExtraConfigSpec{ - Tempo: apiextensionsv1.JSON{Raw: []byte(`{"storage": {"trace": {"wal": {"overlay_setting": "abc"}}}}`)}, + Tempo: apiextensionsv1.JSON{Raw: []byte(`{"storage": {"trace": {"wal": {"overlay_setting": "abc"}}}, "server": {"http_server_read_timeout": "1m", "http_server_write_timeout": "1m"}}`)}, }, }, expected: ` server: http_listen_port: 3200 + http_server_read_timeout: 1m + http_server_write_timeout: 1m internal_server: enable: true http_listen_address: 0.0.0.0 diff --git a/internal/manifests/monolithic/jaegerui_ingress.go b/internal/manifests/monolithic/jaegerui_ingress.go index 67d1c79a1..2ccb66404 100644 --- a/internal/manifests/monolithic/jaegerui_ingress.go +++ b/internal/manifests/monolithic/jaegerui_ingress.go @@ -69,6 +69,8 @@ func BuildJaegerUIIngress(opts Options) *networkingv1.Ingress { return ingress } +const timeoutRouteAnnotation = "haproxy.router.openshift.io/timeout" + // BuildJaegerUIRoute creates a Route object for Jaeger UI. func BuildJaegerUIRoute(opts Options) (*routev1.Route, error) { tempo := opts.Tempo @@ -89,6 +91,14 @@ func BuildJaegerUIRoute(opts Options) (*routev1.Route, error) { return nil, fmt.Errorf("unsupported tls termination '%s' specified for route", tempo.Spec.JaegerUI.Route.Termination) } + annotations := opts.Tempo.Spec.JaegerUI.Route.Annotations + if annotations == nil { + annotations = map[string]string{} + } + if annotations[timeoutRouteAnnotation] == "" { + annotations[timeoutRouteAnnotation] = fmt.Sprintf("%ds", int(tempo.Spec.Timeout.Duration.Seconds())) + } + return &routev1.Route{ TypeMeta: metav1.TypeMeta{ APIVersion: networkingv1.SchemeGroupVersion.String(), @@ -98,7 +108,7 @@ func BuildJaegerUIRoute(opts Options) (*routev1.Route, error) { Name: naming.Name(manifestutils.JaegerUIComponentName, tempo.Name), Namespace: tempo.Namespace, Labels: labels, - Annotations: tempo.Spec.JaegerUI.Route.Annotations, + Annotations: annotations, }, Spec: routev1.RouteSpec{ Host: tempo.Spec.JaegerUI.Route.Host, diff --git a/internal/manifests/monolithic/jaegerui_ingress_test.go b/internal/manifests/monolithic/jaegerui_ingress_test.go index 96824a432..e5cac15ad 100644 --- a/internal/manifests/monolithic/jaegerui_ingress_test.go +++ b/internal/manifests/monolithic/jaegerui_ingress_test.go @@ -163,6 +163,9 @@ func TestBuildJaegerUIRoute(t *testing.T) { Name: "tempo-sample-jaegerui", Namespace: "default", Labels: labels, + Annotations: map[string]string{ + "haproxy.router.openshift.io/timeout": "30s", + }, }, Spec: routev1.RouteSpec{ Host: "", @@ -221,6 +224,9 @@ func TestBuildJaegerUIRoute(t *testing.T) { Name: "tempo-sample-jaegerui", Namespace: "default", Labels: labels, + Annotations: map[string]string{ + "haproxy.router.openshift.io/timeout": "30s", + }, }, Spec: routev1.RouteSpec{ Host: "", diff --git a/internal/manifests/monolithic/statefulset.go b/internal/manifests/monolithic/statefulset.go index 0ba9ae8b6..b62a466b1 100644 --- a/internal/manifests/monolithic/statefulset.go +++ b/internal/manifests/monolithic/statefulset.go @@ -398,6 +398,7 @@ func configureGateway(opts Options, sts *appsv1.StatefulSet) error { fmt.Sprintf("--web.internal.listen=0.0.0.0:%d", manifestutils.GatewayPortInternalHTTPServer), // serves health checks fmt.Sprintf("--traces.tenant-header=%s", manifestutils.TenantHeader), fmt.Sprintf("--traces.tempo.endpoint=http://localhost:%d", manifestutils.PortHTTPServer), // Tempo API upstream + fmt.Sprintf("--traces.write-timeout=%s", opts.Tempo.Spec.Timeout.Duration.String()), fmt.Sprintf("--rbac.config=%s", path.Join(gatewayMountDir, "rbac", manifestutils.GatewayRBACFileName)), fmt.Sprintf("--tenants.config=%s", path.Join(gatewayMountDir, "tenants", manifestutils.GatewayTenantFileName)), "--log.level=info", diff --git a/internal/manifests/monolithic/statefulset_test.go b/internal/manifests/monolithic/statefulset_test.go index b2651704e..aa0239afc 100644 --- a/internal/manifests/monolithic/statefulset_test.go +++ b/internal/manifests/monolithic/statefulset_test.go @@ -2,6 +2,7 @@ package monolithic import ( "testing" + "time" "github.com/operator-framework/operator-lib/proxy" "github.com/stretchr/testify/require" @@ -732,6 +733,7 @@ func TestStatefulsetGateway(t *testing.T) { Namespace: "default", }, Spec: v1alpha1.TempoMonolithicSpec{ + Timeout: metav1.Duration{Duration: time.Second * 5}, Storage: &v1alpha1.MonolithicStorageSpec{ Traces: v1alpha1.MonolithicTracesStorageSpec{ Backend: "memory", @@ -788,6 +790,7 @@ func TestStatefulsetGateway(t *testing.T) { "--web.internal.listen=0.0.0.0:8081", "--traces.tenant-header=x-scope-orgid", "--traces.tempo.endpoint=http://localhost:3200", + "--traces.write-timeout=5s", "--rbac.config=/etc/tempo-gateway/rbac/rbac.yaml", "--tenants.config=/etc/tempo-gateway/tenants/tenants.yaml", "--log.level=info", diff --git a/internal/manifests/oauthproxy/oauth_proxy.go b/internal/manifests/oauthproxy/oauth_proxy.go index 2e90f2f12..f440e9c5c 100644 --- a/internal/manifests/oauthproxy/oauth_proxy.go +++ b/internal/manifests/oauthproxy/oauth_proxy.go @@ -3,6 +3,7 @@ package oauthproxy import ( "fmt" "strings" + "time" routev1 "github.com/openshift/api/route/v1" "github.com/operator-framework/operator-lib/proxy" @@ -69,9 +70,12 @@ func PatchRouteForOauthProxy(route *routev1.Route) { // point route to the oauth } // PatchStatefulSetForOauthProxy returns a modified StatefulSet with the oauth sidecar container and the right service account. -func PatchStatefulSetForOauthProxy(tempo metav1.ObjectMeta, +func PatchStatefulSetForOauthProxy( + tempo metav1.ObjectMeta, authSpec *v1alpha1.JaegerQueryAuthenticationSpec, - config configv1alpha1.ProjectConfig, statefulSet *v1.StatefulSet) { + timeout time.Duration, + config configv1alpha1.ProjectConfig, + statefulSet *v1.StatefulSet) { statefulSet.Spec.Template.Spec.Volumes = append(statefulSet.Spec.Template.Spec.Volumes, corev1.Volume{ Name: getTLSSecretNameForFrontendService(tempo.Name), VolumeSource: corev1.VolumeSource{ @@ -82,7 +86,7 @@ func PatchStatefulSetForOauthProxy(tempo metav1.ObjectMeta, }) statefulSet.Spec.Template.Spec.Containers = append(statefulSet.Spec.Template.Spec.Containers, - oAuthProxyContainer(tempo.Name, statefulSet.Spec.Template.Spec.ServiceAccountName, authSpec, config.DefaultImages.OauthProxy)) + oAuthProxyContainer(tempo.Name, statefulSet.Spec.Template.Spec.ServiceAccountName, authSpec, timeout, config.DefaultImages.OauthProxy)) } // PatchDeploymentForOauthProxy returns a modified deployment with the oauth sidecar container and the right service account. @@ -90,6 +94,7 @@ func PatchDeploymentForOauthProxy( tempo metav1.ObjectMeta, config configv1alpha1.ProjectConfig, authSpec *v1alpha1.JaegerQueryAuthenticationSpec, + timeout time.Duration, imageSpec configv1alpha1.ImagesSpec, dep *v1.Deployment) { dep.Spec.Template.Spec.Volumes = append(dep.Spec.Template.Spec.Volumes, corev1.Volume{ @@ -109,15 +114,18 @@ func PatchDeploymentForOauthProxy( } dep.Spec.Template.Spec.Containers = append(dep.Spec.Template.Spec.Containers, - oAuthProxyContainer(tempo.Name, naming.Name(manifestutils.QueryFrontendComponentName, tempo.Name), - authSpec, oauthProxyImage)) + oAuthProxyContainer(tempo.Name, + naming.Name(manifestutils.QueryFrontendComponentName, tempo.Name), + authSpec, + timeout, + oauthProxyImage)) } func getTLSSecretNameForFrontendService(tempoName string) string { return fmt.Sprintf("%s-ui-oauth-proxy-tls", tempoName) } -func proxyInitArguments(serviceAccountName string) []string { +func proxyInitArguments(serviceAccountName string, timeout time.Duration) []string { return []string{ // The SA Token is injected by admission controller by adding a volume via pod mutation // In Kubernetes 1.24 the SA token is short-lived (default 1h) @@ -136,6 +144,7 @@ func proxyInitArguments(serviceAccountName string) []string { fmt.Sprintf("--tls-cert=%s/tls.crt", tlsProxyPath), fmt.Sprintf("--tls-key=%s/tls.key", tlsProxyPath), fmt.Sprintf("--upstream=http://localhost:%d", manifestutils.PortJaegerUI), + fmt.Sprintf("--upstream-timeout=%s", timeout.String()), } } @@ -143,9 +152,10 @@ func oAuthProxyContainer( tempo string, serviceAccountName string, authSpec *v1alpha1.JaegerQueryAuthenticationSpec, + timeout time.Duration, oauthProxyImage string, ) corev1.Container { - args := proxyInitArguments(serviceAccountName) + args := proxyInitArguments(serviceAccountName, timeout) if len(strings.TrimSpace(authSpec.SAR)) > 0 { args = append(args, fmt.Sprintf("--openshift-sar=%s", authSpec.SAR)) diff --git a/internal/manifests/oauthproxy/oauth_proxy_test.go b/internal/manifests/oauthproxy/oauth_proxy_test.go index bd09a96fe..bde7ff220 100644 --- a/internal/manifests/oauthproxy/oauth_proxy_test.go +++ b/internal/manifests/oauthproxy/oauth_proxy_test.go @@ -2,6 +2,7 @@ package oauthproxy import ( "fmt" + "time" "testing" @@ -45,6 +46,7 @@ func TestOauthProxyContainer(t *testing.T) { fmt.Sprintf("--tls-cert=%s/tls.crt", tlsProxyPath), fmt.Sprintf("--tls-key=%s/tls.key", tlsProxyPath), fmt.Sprintf("--upstream=http://localhost:%d", manifestutils.PortJaegerUI), + "--upstream-timeout=5s", }, tempo: v1alpha1.TempoStack{ ObjectMeta: metav1.ObjectMeta{ @@ -75,6 +77,7 @@ func TestOauthProxyContainer(t *testing.T) { fmt.Sprintf("--tls-cert=%s/tls.crt", tlsProxyPath), fmt.Sprintf("--tls-key=%s/tls.key", tlsProxyPath), fmt.Sprintf("--upstream=http://localhost:%d", manifestutils.PortJaegerUI), + "--upstream-timeout=5s", "--openshift-sar={\"namespace\":\"app-dev\",\"resource\":\"services\",\"resourceName\":\"proxy\",\"verb\":\"get\"}", }, tempo: v1alpha1.TempoStack{ @@ -112,6 +115,7 @@ func TestOauthProxyContainer(t *testing.T) { container := oAuthProxyContainer(params.Tempo.Name, naming.Name(manifestutils.QueryFrontendComponentName, params.Tempo.Name), params.Tempo.Spec.Template.QueryFrontend.JaegerQuery.Authentication, + time.Second*5, customImage, ) expected := corev1.Container{ @@ -340,6 +344,7 @@ func TestPatchDeploymentForOauthProxy(t *testing.T) { params.Tempo.ObjectMeta, params.CtrlConfig, params.Tempo.Spec.Template.QueryFrontend.JaegerQuery.Authentication, + time.Second*5, params.Tempo.Spec.Images, dep) @@ -478,6 +483,7 @@ func TestPatchStatefulSetForOauthProxy(t *testing.T) { PatchStatefulSetForOauthProxy( params.Tempo.ObjectMeta, params.Tempo.Spec.Template.QueryFrontend.JaegerQuery.Authentication, + time.Second*5, params.CtrlConfig, statefulSet) diff --git a/internal/manifests/queryfrontend/query_frontend.go b/internal/manifests/queryfrontend/query_frontend.go index f457135df..27ca29859 100644 --- a/internal/manifests/queryfrontend/query_frontend.go +++ b/internal/manifests/queryfrontend/query_frontend.go @@ -34,6 +34,8 @@ const ( containerNameTempo = "tempo" containerNameJaegerQuery = "jaeger-query" containerNameTempoQuery = "tempo-query" + + timeoutRouteAnnotation = "haproxy.router.openshift.io/timeout" ) // BuildQueryFrontend creates the query-frontend objects. @@ -54,7 +56,7 @@ func BuildQueryFrontend(params manifestutils.Params) ([]client.Object, error) { if gates.HTTPEncryption || gates.GRPCEncryption { caBundleName := naming.SigningCABundleName(tempo.Name) - targets := []string{containerNameTempo, containerNameTempoQuery} + targets := []string{containerNameTempo, containerNameJaegerQuery, containerNameTempoQuery} if err := manifestutils.ConfigureServiceCAByContainerName(&d.Spec.Template.Spec, caBundleName, targets...); err != nil { return nil, err } @@ -85,9 +87,12 @@ func BuildQueryFrontend(params manifestutils.Params) ([]client.Object, error) { if jaegerUIAuthentication != nil && jaegerUIAuthentication.Enabled { oauthproxy.PatchDeploymentForOauthProxy( - tempo.ObjectMeta, params.CtrlConfig, + tempo.ObjectMeta, + params.CtrlConfig, tempo.Spec.Template.QueryFrontend.JaegerQuery.Authentication, - tempo.Spec.Images, d) + tempo.Spec.Timeout.Duration, + tempo.Spec.Images, + d) oauthproxy.PatchQueryFrontEndService(getQueryFrontendService(tempo, svcs), tempo.Name) manifests = append(manifests, oauthproxy.OAuthServiceAccount(params)) @@ -582,12 +587,20 @@ func route(tempo v1alpha1.TempoStack) (*routev1.Route, error) { serviceName := naming.Name(manifestutils.QueryFrontendComponentName, tempo.Name) + annotations := tempo.Spec.Template.QueryFrontend.JaegerQuery.Ingress.Annotations + if annotations == nil { + annotations = map[string]string{} + } + if annotations[timeoutRouteAnnotation] == "" { + annotations[timeoutRouteAnnotation] = fmt.Sprintf("%ds", int(tempo.Spec.Timeout.Duration.Seconds())) + } + return &routev1.Route{ ObjectMeta: metav1.ObjectMeta{ Name: queryFrontendName, Namespace: tempo.Namespace, Labels: labels, - Annotations: tempo.Spec.Template.QueryFrontend.JaegerQuery.Ingress.Annotations, + Annotations: annotations, }, Spec: routev1.RouteSpec{ Host: tempo.Spec.Template.QueryFrontend.JaegerQuery.Ingress.Host, diff --git a/internal/manifests/queryfrontend/query_frontend_test.go b/internal/manifests/queryfrontend/query_frontend_test.go index c11c7f352..e9a5650f9 100644 --- a/internal/manifests/queryfrontend/query_frontend_test.go +++ b/internal/manifests/queryfrontend/query_frontend_test.go @@ -511,6 +511,9 @@ func TestQueryFrontendJaegerRoute(t *testing.T) { Name: naming.Name(manifestutils.QueryFrontendComponentName, "test"), Namespace: "project1", Labels: manifestutils.ComponentLabels("query-frontend", "test"), + Annotations: map[string]string{ + "haproxy.router.openshift.io/timeout": "0s", + }, }, Spec: routev1.RouteSpec{ To: routev1.RouteTargetReference{ @@ -810,6 +813,9 @@ func TestQueryFrontendJaegerRouteSecured(t *testing.T) { Name: naming.Name(manifestutils.QueryFrontendComponentName, "test"), Namespace: "project1", Labels: manifestutils.ComponentLabels("query-frontend", "test"), + Annotations: map[string]string{ + "haproxy.router.openshift.io/timeout": "0s", + }, }, Spec: routev1.RouteSpec{ To: routev1.RouteTargetReference{ diff --git a/internal/webhooks/tempostack_webhook.go b/internal/webhooks/tempostack_webhook.go index 7f6a1f3e1..ea0f944fa 100644 --- a/internal/webhooks/tempostack_webhook.go +++ b/internal/webhooks/tempostack_webhook.go @@ -33,6 +33,7 @@ var ( zeroQuantity = resource.MustParse("0Gi") tenGBQuantity = resource.MustParse("10Gi") defaultServicesDuration = metav1.Duration{Duration: time.Hour * 24 * 3} + defaultTimeout = metav1.Duration{Duration: time.Second * 30} ) // TempoStackWebhook provides webhooks for TempoStack CR. @@ -171,6 +172,10 @@ func (d *Defaulter) Default(ctx context.Context, obj runtime.Object) error { } } + if r.Spec.Timeout.Duration == 0 { + r.Spec.Timeout = defaultTimeout + } + return nil } diff --git a/internal/webhooks/tempostack_webhook_test.go b/internal/webhooks/tempostack_webhook_test.go index b051ab134..b24f02553 100644 --- a/internal/webhooks/tempostack_webhook_test.go +++ b/internal/webhooks/tempostack_webhook_test.go @@ -86,6 +86,7 @@ func TestDefault(t *testing.T) { }, Spec: v1alpha1.TempoStackSpec{ ReplicationFactor: 2, + Timeout: metav1.Duration{Duration: time.Second * 30}, Images: configv1alpha1.ImagesSpec{ Tempo: "docker.io/grafana/tempo:1.2.3", TempoQuery: "docker.io/grafana/tempo-query:1.2.3", @@ -162,6 +163,7 @@ func TestDefault(t *testing.T) { }, Spec: v1alpha1.TempoStackSpec{ ReplicationFactor: 1, + Timeout: metav1.Duration{Duration: time.Second * 30}, Images: configv1alpha1.ImagesSpec{}, ServiceAccount: "tempo-test", Retention: v1alpha1.RetentionSpec{ @@ -244,6 +246,7 @@ func TestDefault(t *testing.T) { }, Spec: v1alpha1.TempoStackSpec{ ReplicationFactor: 1, + Timeout: metav1.Duration{Duration: time.Second * 30}, Images: configv1alpha1.ImagesSpec{}, ServiceAccount: "tempo-test", Retention: v1alpha1.RetentionSpec{ @@ -326,6 +329,7 @@ func TestDefault(t *testing.T) { }, Spec: v1alpha1.TempoStackSpec{ ReplicationFactor: 1, + Timeout: metav1.Duration{Duration: time.Second * 30}, Images: configv1alpha1.ImagesSpec{}, ServiceAccount: "tempo-test", Retention: v1alpha1.RetentionSpec{ @@ -425,6 +429,7 @@ func TestDefault(t *testing.T) { }, Spec: v1alpha1.TempoStackSpec{ ReplicationFactor: 1, + Timeout: metav1.Duration{Duration: time.Second * 30}, Images: configv1alpha1.ImagesSpec{}, ServiceAccount: "tempo-test", Retention: v1alpha1.RetentionSpec{ @@ -491,6 +496,108 @@ func TestDefault(t *testing.T) { Distribution: "upstream", }, }, + { + name: "timeout is set", + input: &v1alpha1.TempoStack{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: v1alpha1.TempoStackSpec{ + ReplicationFactor: 2, + Images: configv1alpha1.ImagesSpec{ + Tempo: "docker.io/grafana/tempo:1.2.3", + TempoQuery: "docker.io/grafana/tempo-query:1.2.3", + TempoGateway: "docker.io/observatorium/gateway:1.2.3", + TempoGatewayOpa: "docker.io/observatorium/opa-openshift:1.2.4", + OauthProxy: "docker.io/observatorium/oauth-proxy:1.2.3", + }, + ServiceAccount: "tempo-test", + Retention: v1alpha1.RetentionSpec{ + Global: v1alpha1.RetentionConfig{ + Traces: metav1.Duration{Duration: time.Hour}, + }, + }, + Timeout: metav1.Duration{Duration: time.Hour}, + StorageSize: resource.MustParse("1Gi"), + LimitSpec: v1alpha1.LimitSpec{ + Global: v1alpha1.RateLimitSpec{ + Query: v1alpha1.QueryLimit{ + MaxSearchDuration: metav1.Duration{Duration: 1 * time.Hour}, + }, + }, + }, + }, + }, + expected: &v1alpha1.TempoStack{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Labels: map[string]string{ + "app.kubernetes.io/managed-by": "tempo-operator", + "tempo.grafana.com/distribution": "upstream", + }, + }, + Spec: v1alpha1.TempoStackSpec{ + ReplicationFactor: 2, + Timeout: metav1.Duration{Duration: time.Hour}, + Images: configv1alpha1.ImagesSpec{ + Tempo: "docker.io/grafana/tempo:1.2.3", + TempoQuery: "docker.io/grafana/tempo-query:1.2.3", + TempoGateway: "docker.io/observatorium/gateway:1.2.3", + TempoGatewayOpa: "docker.io/observatorium/opa-openshift:1.2.4", + OauthProxy: "docker.io/observatorium/oauth-proxy:1.2.3", + }, + ServiceAccount: "tempo-test", + Retention: v1alpha1.RetentionSpec{ + Global: v1alpha1.RetentionConfig{ + Traces: metav1.Duration{Duration: time.Hour}, + }, + }, + StorageSize: resource.MustParse("1Gi"), + LimitSpec: v1alpha1.LimitSpec{ + Global: v1alpha1.RateLimitSpec{ + Query: v1alpha1.QueryLimit{ + MaxSearchDuration: metav1.Duration{Duration: 1 * time.Hour}, + }, + }, + }, + SearchSpec: v1alpha1.SearchSpec{ + MaxDuration: metav1.Duration{Duration: 0}, + DefaultResultLimit: &defaultDefaultResultLimit, + }, + Template: v1alpha1.TempoTemplateSpec{ + Compactor: v1alpha1.TempoComponentSpec{ + Replicas: ptr.To(int32(1)), + }, + Distributor: v1alpha1.TempoDistributorSpec{ + TempoComponentSpec: v1alpha1.TempoComponentSpec{ + Replicas: ptr.To(int32(1)), + }, + TLS: v1alpha1.TLSSpec{}, + }, + Ingester: v1alpha1.TempoComponentSpec{ + Replicas: ptr.To(int32(1)), + }, + Querier: v1alpha1.TempoComponentSpec{ + Replicas: ptr.To(int32(1)), + }, + Gateway: v1alpha1.TempoGatewaySpec{ + TempoComponentSpec: v1alpha1.TempoComponentSpec{ + Replicas: ptr.To(int32(1)), + }, + }, + QueryFrontend: v1alpha1.TempoQueryFrontendSpec{ + TempoComponentSpec: v1alpha1.TempoComponentSpec{ + Replicas: ptr.To(int32(1)), + }, + JaegerQuery: v1alpha1.JaegerQuerySpec{ + ServicesQueryDuration: &defaultServicesDuration, + }, + }, + }, + }, + }, + ctrlConfig: defaultCfgConfig, + }, } for _, test := range tests { diff --git a/tests/e2e-openshift/component-replicas/install-tempo-assert.yaml b/tests/e2e-openshift/component-replicas/install-tempo-assert.yaml index 450abdec7..d306e7429 100644 --- a/tests/e2e-openshift/component-replicas/install-tempo-assert.yaml +++ b/tests/e2e-openshift/component-replicas/install-tempo-assert.yaml @@ -156,6 +156,7 @@ spec: - --web.internal.listen=0.0.0.0:8081 - --traces.write.otlpgrpc.endpoint=tempo-cmpreps-distributor.chainsaw-replicas.svc.cluster.local:4317 - --traces.write.otlphttp.endpoint=https://tempo-cmpreps-distributor.chainsaw-replicas.svc.cluster.local:4318 + - --traces.write-timeout=30s - --traces.tempo.endpoint=https://tempo-cmpreps-query-frontend.chainsaw-replicas.svc.cluster.local:3200 - --grpc.listen=0.0.0.0:8090 - --rbac.config=/etc/tempo-gateway/cm/rbac.yaml diff --git a/tests/e2e-openshift/component-replicas/scale-tempo-assert.yaml b/tests/e2e-openshift/component-replicas/scale-tempo-assert.yaml index c23c4bb52..3e493fa8d 100644 --- a/tests/e2e-openshift/component-replicas/scale-tempo-assert.yaml +++ b/tests/e2e-openshift/component-replicas/scale-tempo-assert.yaml @@ -135,6 +135,7 @@ metadata: kind: TempoStack name: cmpreps spec: + replicas: 2 selector: matchLabels: app.kubernetes.io/component: gateway @@ -156,6 +157,7 @@ spec: - --web.internal.listen=0.0.0.0:8081 - --traces.write.otlpgrpc.endpoint=tempo-cmpreps-distributor.chainsaw-replicas.svc.cluster.local:4317 - --traces.write.otlphttp.endpoint=https://tempo-cmpreps-distributor.chainsaw-replicas.svc.cluster.local:4318 + - --traces.write-timeout=30s - --traces.tempo.endpoint=https://tempo-cmpreps-query-frontend.chainsaw-replicas.svc.cluster.local:3200 - --grpc.listen=0.0.0.0:8090 - --rbac.config=/etc/tempo-gateway/cm/rbac.yaml diff --git a/tests/e2e-openshift/monolithic-multitenancy-static/03-assert.yaml b/tests/e2e-openshift/monolithic-multitenancy-static/03-assert.yaml index 0d3eaf2bc..b1e2dff23 100644 --- a/tests/e2e-openshift/monolithic-multitenancy-static/03-assert.yaml +++ b/tests/e2e-openshift/monolithic-multitenancy-static/03-assert.yaml @@ -2,5 +2,27 @@ apiVersion: apps/v1 kind: Deployment metadata: name: opentelemetry-collector + namespace: chainsaw-monolithic-multitenancy-static status: + availableReplicas: 1 readyReplicas: 1 + replicas: 1 + +--- +apiVersion: v1 +kind: Service +metadata: + name: opentelemetry-collector + namespace: chainsaw-monolithic-multitenancy-static +spec: + ports: + - appProtocol: grpc + name: otlp-grpc + port: 4317 + protocol: TCP + targetPort: 4317 + selector: + app.kubernetes.io/component: opentelemetry-collector + app.kubernetes.io/instance: chainsaw-monolithic-multitenancy-static.opentelemetry + app.kubernetes.io/managed-by: opentelemetry-operator + app.kubernetes.io/part-of: opentelemetry \ No newline at end of file diff --git a/tests/e2e-openshift/monolithic-multitenancy-static/03-install-otel.yaml b/tests/e2e-openshift/monolithic-multitenancy-static/03-install-otel.yaml index b9ed7c52d..3f97a5dca 100644 --- a/tests/e2e-openshift/monolithic-multitenancy-static/03-install-otel.yaml +++ b/tests/e2e-openshift/monolithic-multitenancy-static/03-install-otel.yaml @@ -1,9 +1,10 @@ -apiVersion: v1 -kind: ConfigMap +apiVersion: opentelemetry.io/v1alpha1 +kind: OpenTelemetryCollector metadata: - name: opentelemetry-collector-configmap -data: - config.yaml: | + name: opentelemetry +spec: + mode: deployment + config: | extensions: oauth2client: client_id: tenant1-oidc-client @@ -35,42 +36,4 @@ data: pipelines: traces: exporters: [otlp] - receivers: [otlp] ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: opentelemetry-collector -spec: - selector: - matchLabels: - app: opentelemetry-collector - template: - metadata: - labels: - app: opentelemetry-collector - spec: - containers: - - name: opentelemetry-collector - image: ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib:0.106.1 - command: ["/otelcol-contrib", "--config=/conf/config.yaml"] - volumeMounts: - - mountPath: /conf - name: opentelemetry-collector-configmap - volumes: - - name: opentelemetry-collector-configmap - configMap: - name: opentelemetry-collector-configmap ---- -apiVersion: v1 -kind: Service -metadata: - name: opentelemetry-collector -spec: - type: ClusterIP - ports: - - name: otlp-grpc - port: 4317 - targetPort: 4317 - selector: - app: opentelemetry-collector + receivers: [otlp] \ No newline at end of file diff --git a/tests/e2e-openshift/monolithic-route/chainsaw-test.yaml b/tests/e2e-openshift/monolithic-route/chainsaw-test.yaml index aea3ab58b..410f1d13e 100755 --- a/tests/e2e-openshift/monolithic-route/chainsaw-test.yaml +++ b/tests/e2e-openshift/monolithic-route/chainsaw-test.yaml @@ -1,4 +1,3 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json apiVersion: chainsaw.kyverno.io/v1alpha1 kind: Test metadata: @@ -11,4 +10,9 @@ spec: - apply: file: install-tempo.yaml - assert: - file: install-tempo-assert.yaml \ No newline at end of file + file: install-tempo-assert.yaml + - name: Run the must-gather and verify the contents + try: + - script: + timeout: 5m + content: ./check-must-gahter.sh diff --git a/tests/e2e-openshift/monolithic-route/check-must-gahter.sh b/tests/e2e-openshift/monolithic-route/check-must-gahter.sh new file mode 100755 index 000000000..2757209a3 --- /dev/null +++ b/tests/e2e-openshift/monolithic-route/check-must-gahter.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# Check if must gather directory exists +MUST_GATHER_DIR=/tmp/monolithic-route +mkdir -p $MUST_GATHER_DIR + +# Run the must-gather script +oc adm must-gather --dest-dir=$MUST_GATHER_DIR --image=quay.io/rhn_support_ikanse/tempo-must-gather:latest -- /usr/bin/must-gather --operator-namespace tempo-operator + +# Define required files and directories +REQUIRED_ITEMS=( + "event-filter.html" + "timestamp" + "*sha*/deployment-tempo-operator-controller.yaml" + "*sha*/olm/installplan-install-*.yaml" + "*sha*/olm/clusterserviceversion-tempo-operator-*.yaml" + "*sha*/olm/operator-opentelemetry-product-openshift-opentelemetry-operator.yaml" + "*sha*/olm/operator-tempo-*-tempo-operator.yaml" + "*sha*/olm/subscription-tempo-operator-*-sub.yaml" + "*sha*/namespaces/chainsaw-mono-route/tempomonolithic/mono-route/tempomonolithic-mono-route.yaml" + "*sha*/namespaces/chainsaw-mono-route/tempomonolithic/mono-route/service-tempo-mono-route-jaegerui.yaml" + "*sha*/namespaces/chainsaw-mono-route/tempomonolithic/mono-route/configmap-tempo-mono-route-serving-cabundle.yaml" + "*sha*/namespaces/chainsaw-mono-route/tempomonolithic/mono-route/statefulset-tempo-mono-route.yaml" + "*sha*/namespaces/chainsaw-mono-route/tempomonolithic/mono-route/service-tempo-mono-route.yaml" + "*sha*/namespaces/chainsaw-mono-route/tempomonolithic/mono-route/route-tempo-mono-route-jaegerui.yaml" + "*sha*/namespaces/chainsaw-mono-route/tempomonolithic/mono-route/configmap-tempo-mono-route-config.yaml" + "*sha*/namespaces/chainsaw-mono-route/tempomonolithic/mono-route/serviceaccount-tempo-mono-route.yaml" + "*sha*/tempo-operator-controller-*" +) + +# Verify each required item +for item in "${REQUIRED_ITEMS[@]}"; do + if ! find "$MUST_GATHER_DIR" -path "$MUST_GATHER_DIR/$item" -print -quit | grep -q .; then + echo "Missing: $item" + exit 1 + else + echo "Found: $item" + fi +done + +# Cleanup the must-gather directory +rm -rf $MUST_GATHER_DIR \ No newline at end of file diff --git a/tests/e2e-openshift/monolithic-route/install-tempo-assert.yaml b/tests/e2e-openshift/monolithic-route/install-tempo-assert.yaml index 241609101..b82463b8f 100644 --- a/tests/e2e-openshift/monolithic-route/install-tempo-assert.yaml +++ b/tests/e2e-openshift/monolithic-route/install-tempo-assert.yaml @@ -130,6 +130,7 @@ spec: - --tls-cert=/etc/tls/private/tls.crt - --tls-key=/etc/tls/private/tls.key - --upstream=http://localhost:16686 + - --upstream-timeout=2m0s - '--openshift-sar={"namespace": "chainsaw-mono-route", "resource": "pods", "verb": "get"}' name: oauth-proxy @@ -248,3 +249,67 @@ spec: app.kubernetes.io/instance: mono-route app.kubernetes.io/managed-by: tempo-operator app.kubernetes.io/name: tempo-monolithic + +--- +apiVersion: v1 +data: + tempo-query.yaml: | + address: 127.0.0.1:7777 + backend: 127.0.0.1:3200 + tenant_header_key: x-scope-orgid + services_query_duration: 72h0m0s + find_traces_concurrent_requests: 2 + tempo.yaml: | + server: + http_listen_port: 3200 + http_server_read_timeout: 2m0s + http_server_write_timeout: 2m0s + internal_server: + enable: true + http_listen_address: 0.0.0.0 + storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/blocks + distributor: + receivers: + otlp: + protocols: + grpc: {} + http: {} + usage_report: + reporting_enabled: false +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/component: config + app.kubernetes.io/instance: mono-route + app.kubernetes.io/managed-by: tempo-operator + app.kubernetes.io/name: tempo-monolithic + name: tempo-mono-route-config + +--- +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + annotations: + haproxy.router.openshift.io/timeout: 120s + labels: + app.kubernetes.io/component: jaegerui + app.kubernetes.io/instance: mono-route + app.kubernetes.io/managed-by: tempo-operator + app.kubernetes.io/name: tempo-monolithic + name: tempo-mono-route-jaegerui +spec: + port: + targetPort: oauth-proxy + tls: + termination: reencrypt + to: + kind: Service + name: tempo-mono-route-jaegerui + weight: 100 + wildcardPolicy: None diff --git a/tests/e2e-openshift/monolithic-route/install-tempo.yaml b/tests/e2e-openshift/monolithic-route/install-tempo.yaml index ca7c389c4..860959934 100644 --- a/tests/e2e-openshift/monolithic-route/install-tempo.yaml +++ b/tests/e2e-openshift/monolithic-route/install-tempo.yaml @@ -4,6 +4,7 @@ metadata: name: mono-route namespace: chainsaw-mono-route spec: + timeout: 2m jaegerui: enabled: true route: diff --git a/tests/e2e-openshift/monolithic-single-tenant-auth/install-tempo-assert.yaml b/tests/e2e-openshift/monolithic-single-tenant-auth/install-tempo-assert.yaml index 2ddab9151..d32052933 100644 --- a/tests/e2e-openshift/monolithic-single-tenant-auth/install-tempo-assert.yaml +++ b/tests/e2e-openshift/monolithic-single-tenant-auth/install-tempo-assert.yaml @@ -130,6 +130,7 @@ spec: - --tls-cert=/etc/tls/private/tls.crt - --tls-key=/etc/tls/private/tls.key - --upstream=http://localhost:16686 + - --upstream-timeout=30s - '--openshift-sar={"namespace": "chainsaw-mst", "resource": "pods", "verb": "get"}' name: oauth-proxy diff --git a/tests/e2e-openshift/multitenancy/01-assert.yaml b/tests/e2e-openshift/multitenancy/01-assert.yaml index cb1017286..22581acd0 100644 --- a/tests/e2e-openshift/multitenancy/01-assert.yaml +++ b/tests/e2e-openshift/multitenancy/01-assert.yaml @@ -151,6 +151,7 @@ spec: - --web.internal.listen=0.0.0.0:8081 - --traces.write.otlpgrpc.endpoint=tempo-simplest-distributor.chainsaw-multitenancy.svc.cluster.local:4317 - --traces.write.otlphttp.endpoint=https://tempo-simplest-distributor.chainsaw-multitenancy.svc.cluster.local:4318 + - --traces.write-timeout=30s - --traces.tempo.endpoint=https://tempo-simplest-query-frontend.chainsaw-multitenancy.svc.cluster.local:3200 - --grpc.listen=0.0.0.0:8090 - --rbac.config=/etc/tempo-gateway/cm/rbac.yaml diff --git a/tests/e2e-openshift/route/chainsaw-test.yaml b/tests/e2e-openshift/route/chainsaw-test.yaml index a8751c4b0..c6e031a3f 100755 --- a/tests/e2e-openshift/route/chainsaw-test.yaml +++ b/tests/e2e-openshift/route/chainsaw-test.yaml @@ -4,10 +4,27 @@ kind: Test metadata: name: route spec: + namespace: chainsaw-route steps: + - name: Install Minio storage + try: + - apply: + file: install-storage.yaml + - assert: + file: install-storage-assert.yaml - name: Install TempoStack with ingress type route try: - apply: file: install-tempo.yaml - assert: file: install-tempo-assert.yaml + - name: Check the status of TempoStack + try: + - script: + timeout: 5m + content: kubectl get --namespace $NAMESPACE tempo simplest -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' | grep True + - name: Run the must-gather and verify the contents + try: + - script: + timeout: 5m + content: ./check-must-gahter.sh diff --git a/tests/e2e-openshift/route/check-must-gahter.sh b/tests/e2e-openshift/route/check-must-gahter.sh new file mode 100755 index 000000000..fb3b1b1b8 --- /dev/null +++ b/tests/e2e-openshift/route/check-must-gahter.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +# Check if must gather directory exists +MUST_GATHER_DIR=/tmp/route +mkdir -p $MUST_GATHER_DIR + +# Run the must-gather script +oc adm must-gather --dest-dir=$MUST_GATHER_DIR --image=quay.io/rhn_support_ikanse/tempo-must-gather:latest -- /usr/bin/must-gather --operator-namespace tempo-operator + +# Define required files and directories +REQUIRED_ITEMS=( + "event-filter.html" + "timestamp" + "*sha*/deployment-tempo-operator-controller.yaml" + "*sha*/olm/installplan-install-*" + "*sha*/olm/clusterserviceversion-tempo-operator-*.yaml" + "*sha*/olm/operator-opentelemetry-product-openshift-opentelemetry-operator.yaml" + "*sha*/olm/operator-*-tempo-operator.yaml" + "*sha*/olm/subscription-tempo-operator-*-sub.yaml" + "*sha*/namespaces/chainsaw-route/tempostack/simplest/service-tempo-simplest-distributor.yaml" + "*sha*/namespaces/chainsaw-route/tempostack/simplest/service-tempo-simplest-ingester.yaml" + "*sha*/namespaces/chainsaw-route/tempostack/simplest/deployment-tempo-simplest-distributor.yaml" + "*sha*/namespaces/chainsaw-route/tempostack/simplest/service-tempo-simplest-querier.yaml" + "*sha*/namespaces/chainsaw-route/tempostack/simplest/configmap-tempo-simplest.yaml" + "*sha*/namespaces/chainsaw-route/tempostack/simplest/service-tempo-simplest-compactor.yaml" + "*sha*/namespaces/chainsaw-route/tempostack/simplest/service-tempo-simplest-query-frontend.yaml" + "*sha*/namespaces/chainsaw-route/tempostack/simplest/deployment-tempo-simplest-querier.yaml" + "*sha*/namespaces/chainsaw-route/tempostack/simplest/tempostack-simplest.yaml" + "*sha*/namespaces/chainsaw-route/tempostack/simplest/serviceaccount-tempo-simplest-query-frontend.yaml" + "*sha*/namespaces/chainsaw-route/tempostack/simplest/statefulset-tempo-simplest-ingester.yaml" + "*sha*/namespaces/chainsaw-route/tempostack/simplest/deployment-tempo-simplest-query-frontend.yaml" + "*sha*/namespaces/chainsaw-route/tempostack/simplest/route-tempo-simplest-query-frontend.yaml" + "*sha*/namespaces/chainsaw-route/tempostack/simplest/service-tempo-simplest-gossip-ring.yaml" + "*sha*/namespaces/chainsaw-route/tempostack/simplest/configmap-tempo-simplest-ca-bundle.yaml" + "*sha*/namespaces/chainsaw-route/tempostack/simplest/serviceaccount-tempo-simplest.yaml" + "*sha*/namespaces/chainsaw-route/tempostack/simplest/deployment-tempo-simplest-compactor.yaml" + "*sha*/namespaces/chainsaw-route/tempostack/simplest/service-tempo-simplest-query-frontend-discovery.yaml" + "*sha*/tempo-operator-controller-*" +) + +# Verify each required item +for item in "${REQUIRED_ITEMS[@]}"; do + if ! find "$MUST_GATHER_DIR" -path "$MUST_GATHER_DIR/$item" -print -quit | grep -q .; then + echo "Missing: $item" + exit 1 + else + echo "Found: $item" + fi +done + +# Cleanup the must-gather directory +rm -rf $MUST_GATHER_DIR \ No newline at end of file diff --git a/tests/e2e-openshift/route/install-storage-assert.yaml b/tests/e2e-openshift/route/install-storage-assert.yaml new file mode 100644 index 000000000..afa1f9c34 --- /dev/null +++ b/tests/e2e-openshift/route/install-storage-assert.yaml @@ -0,0 +1,6 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: minio +status: + readyReplicas: 1 diff --git a/tests/e2e-openshift/route/install-storage.yaml b/tests/e2e-openshift/route/install-storage.yaml new file mode 100644 index 000000000..d85862e72 --- /dev/null +++ b/tests/e2e-openshift/route/install-storage.yaml @@ -0,0 +1,75 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + labels: + app.kubernetes.io/name: minio + name: minio +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: minio +spec: + selector: + matchLabels: + app.kubernetes.io/name: minio + strategy: + type: Recreate + template: + metadata: + labels: + app.kubernetes.io/name: minio + spec: + containers: + - command: + - /bin/sh + - -c + - | + mkdir -p /storage/tempo && \ + minio server /storage + env: + - name: MINIO_ACCESS_KEY + value: tempo + - name: MINIO_SECRET_KEY + value: supersecret + image: quay.io/minio/minio:latest + name: minio + ports: + - containerPort: 9000 + volumeMounts: + - mountPath: /storage + name: storage + volumes: + - name: storage + persistentVolumeClaim: + claimName: minio +--- +apiVersion: v1 +kind: Service +metadata: + name: minio +spec: + ports: + - port: 9000 + protocol: TCP + targetPort: 9000 + selector: + app.kubernetes.io/name: minio + type: ClusterIP +--- +apiVersion: v1 +kind: Secret +metadata: + name: minio +stringData: + endpoint: http://minio:9000 + bucket: tempo + access_key_id: tempo + access_key_secret: supersecret +type: Opaque diff --git a/tests/e2e-openshift/route/install-tempo-assert.yaml b/tests/e2e-openshift/route/install-tempo-assert.yaml index 78e16793f..dfe8e64c0 100644 --- a/tests/e2e-openshift/route/install-tempo-assert.yaml +++ b/tests/e2e-openshift/route/install-tempo-assert.yaml @@ -3,6 +3,7 @@ kind: Route metadata: annotations: example_annotation: example_value + haproxy.router.openshift.io/timeout: 120s labels: app.kubernetes.io/component: query-frontend app.kubernetes.io/instance: simplest @@ -19,3 +20,338 @@ spec: kind: Service name: tempo-simplest-query-frontend weight: 100 + +--- +apiVersion: v1 +data: + overrides.yaml: | + overrides: + tempo-query-frontend.yaml: | + compactor: + compaction: + block_retention: 48h0m0s + ring: + kvstore: + store: memberlist + distributor: + receivers: + jaeger: + protocols: + thrift_http: + endpoint: 0.0.0.0:14268 + thrift_binary: + endpoint: 0.0.0.0:6832 + thrift_compact: + endpoint: 0.0.0.0:6831 + grpc: + endpoint: 0.0.0.0:14250 + zipkin: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + ring: + kvstore: + store: memberlist + ingester: + lifecycler: + ring: + kvstore: + store: memberlist + replication_factor: 1 + tokens_file_path: /var/tempo/tokens.json + max_block_duration: 10m + memberlist: + abort_if_cluster_join_fails: false + join_members: + - tempo-simplest-gossip-ring + multitenancy_enabled: false + querier: + max_concurrent_queries: 20 + frontend_worker: + frontend_address: tempo-simplest-query-frontend-discovery:9095 + grpc_client_config: + tls_enabled: true + tls_cert_path: /var/run/tls/server/tls.crt + tls_key_path: /var/run/tls/server/tls.key + tls_ca_path: /var/run/ca/service-ca.crt + tls_server_name: tempo-simplest-query-frontend.chainsaw-route.svc.cluster.local + tls_min_version: VersionTLS13 + search: + external_hedge_requests_at: 8s + external_hedge_requests_up_to: 2 + server: + grpc_server_max_recv_msg_size: 4194304 + grpc_server_max_send_msg_size: 4194304 + http_listen_port: 3200 + http_server_read_timeout: 2m0s + http_server_write_timeout: 2m0s + log_format: logfmt + tls_min_version: VersionTLS13 + grpc_tls_config: + cert_file: /var/run/tls/server/tls.crt + key_file: /var/run/tls/server/tls.key + client_ca_file: /var/run/ca/service-ca.crt + client_auth_type: RequireAndVerifyClientCert + storage: + trace: + backend: s3 + blocklist_poll: 5m + cache: none + s3: + endpoint: minio:9000 + bucket: tempo + insecure: true + local: + path: /var/tempo/traces + wal: + path: /var/tempo/wal + usage_report: + reporting_enabled: false + query_frontend: + search: + concurrent_jobs: 2000 + max_duration: 0s + default_result_limit: 20 + ingester_client: + grpc_client_config: + tls_enabled: true + tls_cert_path: /var/run/tls/server/tls.crt + tls_key_path: /var/run/tls/server/tls.key + tls_ca_path: /var/run/ca/service-ca.crt + tls_server_name: tempo-simplest-ingester.chainsaw-route.svc.cluster.local + tls_insecure_skip_verify: false + tls_min_version: VersionTLS13 + tempo-query.yaml: | + address: 127.0.0.1:7777 + backend: 127.0.0.1:3200 + tenant_header_key: x-scope-orgid + services_query_duration: 72h0m0s + find_traces_concurrent_requests: 2 + tempo.yaml: | + compactor: + compaction: + block_retention: 48h0m0s + ring: + kvstore: + store: memberlist + distributor: + receivers: + jaeger: + protocols: + thrift_http: + endpoint: 0.0.0.0:14268 + thrift_binary: + endpoint: 0.0.0.0:6832 + thrift_compact: + endpoint: 0.0.0.0:6831 + grpc: + endpoint: 0.0.0.0:14250 + zipkin: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + ring: + kvstore: + store: memberlist + ingester: + lifecycler: + ring: + kvstore: + store: memberlist + replication_factor: 1 + tokens_file_path: /var/tempo/tokens.json + max_block_duration: 10m + memberlist: + abort_if_cluster_join_fails: false + join_members: + - tempo-simplest-gossip-ring + multitenancy_enabled: false + querier: + max_concurrent_queries: 20 + frontend_worker: + frontend_address: tempo-simplest-query-frontend-discovery:9095 + grpc_client_config: + tls_enabled: true + tls_cert_path: /var/run/tls/server/tls.crt + tls_key_path: /var/run/tls/server/tls.key + tls_ca_path: /var/run/ca/service-ca.crt + tls_server_name: tempo-simplest-query-frontend.chainsaw-route.svc.cluster.local + tls_min_version: VersionTLS13 + search: + external_hedge_requests_at: 8s + external_hedge_requests_up_to: 2 + internal_server: + enable: true + http_listen_address: "" + tls_min_version: VersionTLS13 + http_tls_config: + cert_file: /var/run/tls/server/tls.crt + key_file: /var/run/tls/server/tls.key + server: + grpc_server_max_recv_msg_size: 4194304 + grpc_server_max_send_msg_size: 4194304 + http_listen_port: 3200 + http_server_read_timeout: 2m0s + http_server_write_timeout: 2m0s + log_format: logfmt + tls_min_version: VersionTLS13 + grpc_tls_config: + cert_file: /var/run/tls/server/tls.crt + key_file: /var/run/tls/server/tls.key + client_ca_file: /var/run/ca/service-ca.crt + client_auth_type: RequireAndVerifyClientCert + http_tls_config: + cert_file: /var/run/tls/server/tls.crt + client_auth_type: RequireAndVerifyClientCert + key_file: /var/run/tls/server/tls.key + client_ca_file: /var/run/ca/service-ca.crt + storage: + trace: + backend: s3 + blocklist_poll: 5m + cache: none + s3: + endpoint: minio:9000 + bucket: tempo + insecure: true + local: + path: /var/tempo/traces + wal: + path: /var/tempo/wal + usage_report: + reporting_enabled: false + query_frontend: + search: + concurrent_jobs: 2000 + max_duration: 0s + default_result_limit: 20 + ingester_client: + grpc_client_config: + tls_enabled: true + tls_cert_path: /var/run/tls/server/tls.crt + tls_key_path: /var/run/tls/server/tls.key + tls_ca_path: /var/run/ca/service-ca.crt + tls_server_name: tempo-simplest-ingester.chainsaw-route.svc.cluster.local + tls_insecure_skip_verify: false + tls_min_version: VersionTLS13 +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/component: config + app.kubernetes.io/instance: simplest + app.kubernetes.io/managed-by: tempo-operator + app.kubernetes.io/name: tempo + name: tempo-simplest + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/component: compactor + app.kubernetes.io/instance: simplest + app.kubernetes.io/managed-by: tempo-operator + app.kubernetes.io/name: tempo + name: tempo-simplest-compactor +status: + availableReplicas: 1 + readyReplicas: 1 + replicas: 1 + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/component: distributor + app.kubernetes.io/instance: simplest + app.kubernetes.io/managed-by: tempo-operator + app.kubernetes.io/name: tempo + name: tempo-simplest-distributor +status: + availableReplicas: 1 + readyReplicas: 1 + replicas: 1 + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/component: querier + app.kubernetes.io/instance: simplest + app.kubernetes.io/managed-by: tempo-operator + app.kubernetes.io/name: tempo + name: tempo-simplest-querier +status: + availableReplicas: 1 + readyReplicas: 1 + replicas: 1 + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/component: query-frontend + app.kubernetes.io/instance: simplest + app.kubernetes.io/managed-by: tempo-operator + app.kubernetes.io/name: tempo + name: tempo-simplest-query-frontend +spec: + selector: + matchLabels: + app.kubernetes.io/component: query-frontend + app.kubernetes.io/instance: simplest + app.kubernetes.io/managed-by: tempo-operator + app.kubernetes.io/name: tempo + template: + metadata: + labels: + app.kubernetes.io/component: query-frontend + app.kubernetes.io/instance: simplest + app.kubernetes.io/managed-by: tempo-operator + app.kubernetes.io/name: tempo + tempo-gossip-member: "true" + spec: + containers: + - name: tempo + - name: jaeger-query + - name: tempo-query + - args: + - --cookie-secret-file=/var/run/secrets/kubernetes.io/serviceaccount/token + - --https-address=:8443 + - --openshift-service-account=tempo-simplest-query-frontend + - --provider=openshift + - --tls-cert=/etc/tls/private/tls.crt + - --tls-key=/etc/tls/private/tls.key + - --upstream=http://localhost:16686 + - --upstream-timeout=2m0s + - '--openshift-sar={"namespace": "chainsaw-route", "resource": "pods", "verb": "get"}' + name: oauth-proxy +status: + availableReplicas: 1 + readyReplicas: 1 + replicas: 1 + +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + app.kubernetes.io/component: ingester + app.kubernetes.io/instance: simplest + app.kubernetes.io/managed-by: tempo-operator + app.kubernetes.io/name: tempo + name: tempo-simplest-ingester +status: + availableReplicas: 1 + currentReplicas: 1 + readyReplicas: 1 + replicas: 1 \ No newline at end of file diff --git a/tests/e2e-openshift/route/install-tempo.yaml b/tests/e2e-openshift/route/install-tempo.yaml index 0feb8af11..ac9556089 100644 --- a/tests/e2e-openshift/route/install-tempo.yaml +++ b/tests/e2e-openshift/route/install-tempo.yaml @@ -16,9 +16,10 @@ kind: TempoStack metadata: name: simplest spec: + timeout: 2m storage: secret: - name: minio-test + name: minio type: s3 storageSize: 200M template: diff --git a/tests/e2e-openshift/tempo-single-tenant-auth/install-tempo-assert.yaml b/tests/e2e-openshift/tempo-single-tenant-auth/install-tempo-assert.yaml index 40f3dcafe..9b6f74c5b 100644 --- a/tests/e2e-openshift/tempo-single-tenant-auth/install-tempo-assert.yaml +++ b/tests/e2e-openshift/tempo-single-tenant-auth/install-tempo-assert.yaml @@ -47,7 +47,25 @@ spec: tempo-gossip-member: "true" spec: containers: - - name: tempo + - args: + - -target=query-frontend + - -config.file=/conf/tempo-query-frontend.yaml + - -mem-ballast-size-mbs=1024 + - -log.level=info + - --storage.trace.s3.secret_key=$(S3_SECRET_KEY) + - --storage.trace.s3.access_key=$(S3_ACCESS_KEY) + env: + - name: S3_SECRET_KEY + valueFrom: + secretKeyRef: + key: access_key_secret + name: minio + - name: S3_ACCESS_KEY + valueFrom: + secretKeyRef: + key: access_key_id + name: minio + name: tempo ports: - containerPort: 3200 name: http @@ -65,7 +83,12 @@ spec: name: tempo-tempo-st-ca-bundle - mountPath: /var/run/tls/server name: tempo-tempo-st-query-frontend-mtls - - name: jaeger-query + - args: + - --query.base-path=/ + - --span-storage.type=grpc + - --grpc-storage.server=localhost:7777 + - --query.bearer-token-propagation=true + name: jaeger-query ports: - containerPort: 16685 name: jaeger-grpc @@ -79,7 +102,13 @@ spec: volumeMounts: - mountPath: /tmp name: tempo-tmp-storage-query - - name: tempo-query + - mountPath: /var/run/ca + name: tempo-tempo-st-ca-bundle + - mountPath: /var/run/tls/server + name: tempo-tempo-st-query-frontend-mtls + - args: + - -config=/conf/tempo-query.yaml + name: tempo-query ports: - containerPort: 7777 name: proxy-grpc @@ -92,7 +121,18 @@ spec: name: tempo-tempo-st-ca-bundle - mountPath: /var/run/tls/server name: tempo-tempo-st-query-frontend-mtls - - name: oauth-proxy + - args: + - --cookie-secret-file=/var/run/secrets/kubernetes.io/serviceaccount/token + - --https-address=:8443 + - --openshift-service-account=tempo-tempo-st-query-frontend + - --provider=openshift + - --tls-cert=/etc/tls/private/tls.crt + - --tls-key=/etc/tls/private/tls.key + - --upstream=http://localhost:16686 + - --upstream-timeout=30s + - '--openshift-sar={"namespace": "chainsaw-mst", "resource": "pods", "verb": + "get"}' + name: oauth-proxy ports: - containerPort: 8443 name: oauth-proxy diff --git a/tests/e2e-openshift/tls-monolithic-singletenant/01-assert.yaml b/tests/e2e-openshift/tls-monolithic-singletenant/01-assert.yaml index 0b1f9dd38..c882016ee 100644 --- a/tests/e2e-openshift/tls-monolithic-singletenant/01-assert.yaml +++ b/tests/e2e-openshift/tls-monolithic-singletenant/01-assert.yaml @@ -136,6 +136,7 @@ spec: - --tls-cert=/etc/tls/private/tls.crt - --tls-key=/etc/tls/private/tls.key - --upstream=http://localhost:16686 + - --upstream-timeout=30s - '--openshift-sar={"namespace": "chainsaw-tls-mono-st", "resource": "pods", "verb": "get"}' name: oauth-proxy @@ -275,6 +276,8 @@ spec: apiVersion: route.openshift.io/v1 kind: Route metadata: + annotations: + haproxy.router.openshift.io/timeout: 30s labels: app.kubernetes.io/component: jaegerui app.kubernetes.io/instance: mono @@ -290,3 +293,53 @@ spec: to: kind: Service name: tempo-mono-jaegerui + +--- +apiVersion: v1 +data: + tempo-query.yaml: | + address: 127.0.0.1:7777 + backend: 127.0.0.1:3200 + tenant_header_key: x-scope-orgid + services_query_duration: 72h0m0s + find_traces_concurrent_requests: 2 + tempo.yaml: | + server: + http_listen_port: 3200 + http_server_read_timeout: 30s + http_server_write_timeout: 30s + internal_server: + enable: true + http_listen_address: 0.0.0.0 + storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/blocks + distributor: + receivers: + otlp: + protocols: + grpc: + tls: + cert_file: /var/run/tls/receiver/grpc/tls.crt + key_file: /var/run/tls/receiver/grpc/tls.key + min_version: "1.3" + http: + tls: + cert_file: /var/run/tls/receiver/http/tls.crt + key_file: /var/run/tls/receiver/http/tls.key + min_version: "1.3" + usage_report: + reporting_enabled: false +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/component: config + app.kubernetes.io/instance: mono + app.kubernetes.io/managed-by: tempo-operator + app.kubernetes.io/name: tempo-monolithic + name: tempo-mono-config + diff --git a/tests/e2e-openshift/tls-singletenant/01-assert.yaml b/tests/e2e-openshift/tls-singletenant/01-assert.yaml index 9b312a0e2..b2202982a 100644 --- a/tests/e2e-openshift/tls-singletenant/01-assert.yaml +++ b/tests/e2e-openshift/tls-singletenant/01-assert.yaml @@ -42,7 +42,25 @@ spec: tempo-gossip-member: "true" spec: containers: - - name: tempo + - args: + - -target=query-frontend + - -config.file=/conf/tempo-query-frontend.yaml + - -mem-ballast-size-mbs=1024 + - -log.level=info + - --storage.trace.s3.secret_key=$(S3_SECRET_KEY) + - --storage.trace.s3.access_key=$(S3_ACCESS_KEY) + env: + - name: S3_SECRET_KEY + valueFrom: + secretKeyRef: + key: access_key_secret + name: minio + - name: S3_ACCESS_KEY + valueFrom: + secretKeyRef: + key: access_key_id + name: minio + name: tempo ports: - containerPort: 3200 name: http @@ -60,7 +78,12 @@ spec: name: tempo-simplest-ca-bundle - mountPath: /var/run/tls/server name: tempo-simplest-query-frontend-mtls - - name: jaeger-query + - args: + - --query.base-path=/ + - --span-storage.type=grpc + - --grpc-storage.server=localhost:7777 + - --query.bearer-token-propagation=true + name: jaeger-query ports: - containerPort: 16685 name: jaeger-grpc @@ -74,7 +97,13 @@ spec: volumeMounts: - mountPath: /tmp name: tempo-tmp-storage-query - - name: tempo-query + - mountPath: /var/run/ca + name: tempo-simplest-ca-bundle + - mountPath: /var/run/tls/server + name: tempo-simplest-query-frontend-mtls + - args: + - -config=/conf/tempo-query.yaml + name: tempo-query ports: - containerPort: 7777 name: proxy-grpc @@ -87,6 +116,22 @@ spec: name: tempo-simplest-ca-bundle - mountPath: /var/run/tls/server name: tempo-simplest-query-frontend-mtls + - args: + - --cookie-secret-file=/var/run/secrets/kubernetes.io/serviceaccount/token + - --https-address=:8443 + - --openshift-service-account=tempo-simplest-query-frontend + - --provider=openshift + - --tls-cert=/etc/tls/private/tls.crt + - --tls-key=/etc/tls/private/tls.key + - --upstream=http://localhost:16686 + - --upstream-timeout=30s + - '--openshift-sar={"namespace": "chainsaw-tls-singletenant", "resource": + "pods", "verb": "get"}' + name: oauth-proxy + ports: + - containerPort: 8443 + name: oauth-proxy + protocol: TCP volumes: - configMap: defaultMode: 420 @@ -104,15 +149,134 @@ spec: secret: defaultMode: 420 secretName: tempo-simplest-query-frontend-mtls + - name: simplest-ui-oauth-proxy-tls + secret: + defaultMode: 420 + secretName: simplest-ui-oauth-proxy-tls status: availableReplicas: 1 readyReplicas: 1 replicas: 1 + --- -apiVersion: apps/v1 -kind: StatefulSet +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + annotations: + haproxy.router.openshift.io/timeout: 30s + labels: + app.kubernetes.io/component: query-frontend + app.kubernetes.io/instance: simplest + app.kubernetes.io/managed-by: tempo-operator + app.kubernetes.io/name: tempo + name: tempo-simplest-query-frontend +spec: + port: + targetPort: oauth-proxy + tls: + termination: reencrypt + to: + kind: Service + name: tempo-simplest-query-frontend + weight: 100 + wildcardPolicy: None + +--- +apiVersion: v1 +data: + overrides.yaml: | + overrides: + tempo-query-frontend.yaml: "compactor:\n compaction:\n block_retention: 48h0m0s\n + \ ring:\n kvstore:\n store: memberlist\ndistributor:\n receivers:\n jaeger:\n + \ protocols:\n thrift_http:\n endpoint: 0.0.0.0:14268\n tls:\n + \ cert_file: /var/run/tls/receiver/tls.crt\n key_file: /var/run/tls/receiver/tls.key\n + \ min_version: \n thrift_binary:\n endpoint: 0.0.0.0:6832\n + \ thrift_compact:\n endpoint: 0.0.0.0:6831\n grpc:\n endpoint: + 0.0.0.0:14250\n tls:\n cert_file: /var/run/tls/receiver/tls.crt\n + \ key_file: /var/run/tls/receiver/tls.key\n min_version: + \n zipkin:\n tls:\n cert_file: /var/run/tls/receiver/tls.crt\n + \ key_file: /var/run/tls/receiver/tls.key\n min_version: \n otlp:\n + \ protocols:\n grpc:\n endpoint: 0.0.0.0:4317\n tls:\n + \ cert_file: /var/run/tls/receiver/tls.crt\n key_file: /var/run/tls/receiver/tls.key\n + \ min_version: \n http:\n endpoint: 0.0.0.0:4318\n tls:\n + \ cert_file: /var/run/tls/receiver/tls.crt\n key_file: /var/run/tls/receiver/tls.key\n + \ min_version: \n ring:\n kvstore:\n store: memberlist\ningester:\n + \ lifecycler:\n ring:\n kvstore:\n store: memberlist\n replication_factor: + 1\n tokens_file_path: /var/tempo/tokens.json\n max_block_duration: 10m\nmemberlist:\n + \ abort_if_cluster_join_fails: false\n join_members:\n - tempo-simplest-gossip-ring\nmultitenancy_enabled: + false\nquerier:\n max_concurrent_queries: 20\n frontend_worker:\n frontend_address: + tempo-simplest-query-frontend-discovery:9095\n grpc_client_config:\n tls_enabled: + true\n tls_cert_path: /var/run/tls/server/tls.crt\n tls_key_path: /var/run/tls/server/tls.key\n + \ tls_ca_path: /var/run/ca/service-ca.crt\n tls_server_name: tempo-simplest-query-frontend.chainsaw-tls-singletenant.svc.cluster.local\n + \ tls_min_version: VersionTLS13\n search:\n external_hedge_requests_at: + 8s\n external_hedge_requests_up_to: 2\nserver:\n grpc_server_max_recv_msg_size: + 4194304\n grpc_server_max_send_msg_size: 4194304\n http_listen_port: 3200\n + \ http_server_read_timeout: 30s\n http_server_write_timeout: 30s\n log_format: + logfmt\n tls_min_version: VersionTLS13\n grpc_tls_config:\n cert_file: /var/run/tls/server/tls.crt\n + \ key_file: /var/run/tls/server/tls.key\n client_ca_file: /var/run/ca/service-ca.crt\n + \ client_auth_type: RequireAndVerifyClientCert\nstorage:\n trace:\n backend: + s3\n blocklist_poll: 5m\n cache: none\n s3:\n endpoint: minio:9000\n + \ bucket: tempo\n insecure: true\n local:\n path: /var/tempo/traces\n + \ wal:\n path: /var/tempo/wal\nusage_report:\n reporting_enabled: false\nquery_frontend:\n + \ search:\n concurrent_jobs: 2000\n max_duration: 0s\n default_result_limit: + 20\ningester_client:\n grpc_client_config:\n tls_enabled: true\n tls_cert_path: + \ /var/run/tls/server/tls.crt\n tls_key_path: /var/run/tls/server/tls.key\n + \ tls_ca_path: /var/run/ca/service-ca.crt\n tls_server_name: tempo-simplest-ingester.chainsaw-tls-singletenant.svc.cluster.local\n + \ tls_insecure_skip_verify: false\n tls_min_version: VersionTLS13\n" + tempo-query.yaml: | + address: 127.0.0.1:7777 + backend: 127.0.0.1:3200 + tenant_header_key: x-scope-orgid + services_query_duration: 72h0m0s + find_traces_concurrent_requests: 2 + tempo.yaml: "compactor:\n compaction:\n block_retention: 48h0m0s\n ring:\n + \ kvstore:\n store: memberlist\ndistributor:\n receivers:\n jaeger:\n + \ protocols:\n thrift_http:\n endpoint: 0.0.0.0:14268\n tls:\n + \ cert_file: /var/run/tls/receiver/tls.crt\n key_file: /var/run/tls/receiver/tls.key\n + \ min_version: \n thrift_binary:\n endpoint: 0.0.0.0:6832\n + \ thrift_compact:\n endpoint: 0.0.0.0:6831\n grpc:\n endpoint: + 0.0.0.0:14250\n tls:\n cert_file: /var/run/tls/receiver/tls.crt\n + \ key_file: /var/run/tls/receiver/tls.key\n min_version: + \n zipkin:\n tls:\n cert_file: /var/run/tls/receiver/tls.crt\n + \ key_file: /var/run/tls/receiver/tls.key\n min_version: \n otlp:\n + \ protocols:\n grpc:\n endpoint: 0.0.0.0:4317\n tls:\n + \ cert_file: /var/run/tls/receiver/tls.crt\n key_file: /var/run/tls/receiver/tls.key\n + \ min_version: \n http:\n endpoint: 0.0.0.0:4318\n tls:\n + \ cert_file: /var/run/tls/receiver/tls.crt\n key_file: /var/run/tls/receiver/tls.key\n + \ min_version: \n ring:\n kvstore:\n store: memberlist\ningester:\n + \ lifecycler:\n ring:\n kvstore:\n store: memberlist\n replication_factor: + 1\n tokens_file_path: /var/tempo/tokens.json\n max_block_duration: 10m\nmemberlist:\n + \ abort_if_cluster_join_fails: false\n join_members:\n - tempo-simplest-gossip-ring\nmultitenancy_enabled: + false\nquerier:\n max_concurrent_queries: 20\n frontend_worker:\n frontend_address: + tempo-simplest-query-frontend-discovery:9095\n grpc_client_config:\n tls_enabled: + true\n tls_cert_path: /var/run/tls/server/tls.crt\n tls_key_path: /var/run/tls/server/tls.key\n + \ tls_ca_path: /var/run/ca/service-ca.crt\n tls_server_name: tempo-simplest-query-frontend.chainsaw-tls-singletenant.svc.cluster.local\n + \ tls_min_version: VersionTLS13\n search:\n external_hedge_requests_at: + 8s\n external_hedge_requests_up_to: 2\ninternal_server:\n enable: true\n http_listen_address: + \"\"\n tls_min_version: VersionTLS13\n http_tls_config:\n cert_file: /var/run/tls/server/tls.crt\n + \ key_file: /var/run/tls/server/tls.key\nserver:\n grpc_server_max_recv_msg_size: + 4194304\n grpc_server_max_send_msg_size: 4194304\n http_listen_port: 3200\n + \ http_server_read_timeout: 30s\n http_server_write_timeout: 30s\n log_format: + logfmt\n tls_min_version: VersionTLS13\n grpc_tls_config:\n cert_file: /var/run/tls/server/tls.crt\n + \ key_file: /var/run/tls/server/tls.key\n client_ca_file: /var/run/ca/service-ca.crt\n + \ client_auth_type: RequireAndVerifyClientCert\n http_tls_config:\n cert_file: + \ /var/run/tls/server/tls.crt\n client_auth_type: RequireAndVerifyClientCert\n + \ key_file: /var/run/tls/server/tls.key\n client_ca_file: /var/run/ca/service-ca.crt\nstorage:\n + \ trace:\n backend: s3\n blocklist_poll: 5m\n cache: none\n s3:\n + \ endpoint: minio:9000\n bucket: tempo\n insecure: true\n local:\n + \ path: /var/tempo/traces\n wal:\n path: /var/tempo/wal\nusage_report:\n + \ reporting_enabled: false\nquery_frontend:\n search:\n concurrent_jobs: 2000\n + \ max_duration: 0s\n default_result_limit: 20\ningester_client:\n grpc_client_config:\n + \ tls_enabled: true\n tls_cert_path: /var/run/tls/server/tls.crt\n tls_key_path: + /var/run/tls/server/tls.key\n tls_ca_path: /var/run/ca/service-ca.crt\n tls_server_name: + tempo-simplest-ingester.chainsaw-tls-singletenant.svc.cluster.local\n tls_insecure_skip_verify: + false\n tls_min_version: VersionTLS13\n" +kind: ConfigMap metadata: - name: tempo-simplest-ingester + labels: + app.kubernetes.io/component: config + app.kubernetes.io/instance: simplest + app.kubernetes.io/managed-by: tempo-operator + app.kubernetes.io/name: tempo + name: tempo-simplest namespace: chainsaw-tls-singletenant -status: - readyReplicas: 1 diff --git a/tests/e2e-openshift/tls-singletenant/01-install-tempo.yaml b/tests/e2e-openshift/tls-singletenant/01-install-tempo.yaml index ebb1d61e3..84e2a2571 100644 --- a/tests/e2e-openshift/tls-singletenant/01-install-tempo.yaml +++ b/tests/e2e-openshift/tls-singletenant/01-install-tempo.yaml @@ -1,4 +1,3 @@ -# based on config/samples/openshift/tempo_v1alpha1_multitenancy.yaml apiVersion: tempo.grafana.com/v1alpha1 kind: TempoStack metadata: @@ -22,3 +21,5 @@ spec: queryFrontend: jaegerQuery: enabled: true + ingress: + type: route diff --git a/tests/e2e/monolithic-extraconfig/install-tempo-assert.yaml b/tests/e2e/monolithic-extraconfig/install-tempo-assert.yaml index d9897a059..883e687d7 100644 --- a/tests/e2e/monolithic-extraconfig/install-tempo-assert.yaml +++ b/tests/e2e/monolithic-extraconfig/install-tempo-assert.yaml @@ -81,6 +81,7 @@ data: backend: 127.0.0.1:3200 tenant_header_key: x-scope-orgid services_query_duration: 72h0m0s + find_traces_concurrent_requests: 2 tempo.yaml: | distributor: receivers: @@ -98,6 +99,8 @@ data: max_retries: 3 server: http_listen_port: 3200 + http_server_read_timeout: 30s + http_server_write_timeout: 30s storage: trace: backend: local diff --git a/tests/e2e/tempostack-extraconfig/install-tempostack-assert.yaml b/tests/e2e/tempostack-extraconfig/install-tempostack-assert.yaml index aa23663bf..6b7d3b59c 100644 --- a/tests/e2e/tempostack-extraconfig/install-tempostack-assert.yaml +++ b/tests/e2e/tempostack-extraconfig/install-tempostack-assert.yaml @@ -456,8 +456,8 @@ data: client_ca_file: /var/run/ca/service-ca.crt key_file: /var/run/tls/server/tls.key http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 10m + http_server_write_timeout: 10m log_format: logfmt tls_min_version: VersionTLS13 storage: @@ -480,6 +480,7 @@ data: backend: 127.0.0.1:3200 tenant_header_key: x-scope-orgid services_query_duration: 72h0m0s + find_traces_concurrent_requests: 2 tempo.yaml: | compactor: compaction: @@ -568,8 +569,8 @@ data: client_ca_file: /var/run/ca/service-ca.crt key_file: /var/run/tls/server/tls.key http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 10m + http_server_write_timeout: 10m http_tls_config: cert_file: /var/run/tls/server/tls.crt client_auth_type: RequireAndVerifyClientCert diff --git a/tests/e2e/tempostack-extraconfig/install-tempostack.yaml b/tests/e2e/tempostack-extraconfig/install-tempostack.yaml index 812ff0bc3..2de1a58a6 100644 --- a/tests/e2e/tempostack-extraconfig/install-tempostack.yaml +++ b/tests/e2e/tempostack-extraconfig/install-tempostack.yaml @@ -3,8 +3,12 @@ kind: TempoStack metadata: name: simplest spec: + timeout: 70s extraConfig: tempo: + server: + http_server_write_timeout: 10m + http_server_read_timeout: 10m querier: search: query_timeout: 180s