From 57400a8613ceb16675ef8ce2f8393f62f5f40ec7 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 8 May 2024 20:09:41 -0400 Subject: [PATCH] remove redundant reference to tail-sampling. It's already covered earlier in the doc that these considerations should be made --- .../deployment/multiple-collectors.md | 8 +- content/en/docs/collector/scaling.md | 114 ++++++------------ content/en/docs/concepts/sampling/index.md | 8 ++ 3 files changed, 51 insertions(+), 79 deletions(-) diff --git a/content/en/docs/collector/deployment/multiple-collectors.md b/content/en/docs/collector/deployment/multiple-collectors.md index 45079005d96b..3b0477d26b5d 100644 --- a/content/en/docs/collector/deployment/multiple-collectors.md +++ b/content/en/docs/collector/deployment/multiple-collectors.md @@ -20,10 +20,10 @@ report on the same data can lead to data loss or, at least, degraded data quality. In gateway collector deployments, applying this principle guards against sending inconsistent data to the backend. All metric data streams within OTLP must have a -[single writer](/docs/specs/otel/metrics/data-model/#single-writer). -In a system with multiple collectors, the single-writer principle is most -relevant for receivers that create their own metrics, such a pull-based scrapers -or a host metrics receiver. +[single writer](/docs/specs/otel/metrics/data-model/#single-writer). In a system +with multiple collectors, the single-writer principle is most relevant for +receivers that create their own metrics, such a pull-based scrapers or a host +metrics receiver. ### Deployment Considerations diff --git a/content/en/docs/collector/scaling.md b/content/en/docs/collector/scaling.md index 25bdbf14e455..4638a75ed9ff 100644 --- a/content/en/docs/collector/scaling.md +++ b/content/en/docs/collector/scaling.md @@ -389,36 +389,6 @@ The Single-Writer Principle refers to employing a single logical writer for a particular resource. When scaling collectors horizontally in a system it's important to properly distinguish between targets using unique identities. -##### Tail-based Sampling - -Partial or incomplete traces may be consequential for an implementation of -tail-sampling, as the goal is to capture all or most of the spans within the -trace in order to inform sampling decisions. When using the target allocator to -identify targets for scraping, you can also add labels. - -```yaml -- job_name: 'otel-collector' - static_configs: - - targets: ['0.0.0.0:8888'] - labels: - service: 'my-service' -``` - -You can also use -[relabel](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config) -to manage labeling on jobs. - -```yaml -job_name: 'otel-collector' -static_configs: - - targets: ['0.0.0.0:8888'] - labels: - service: 'my-service' - relabel_configs: - - source_labels: [service] - target_label: app - action: replace -``` ##### Pull-based Scraping @@ -429,52 +399,46 @@ lower, you may be able to rely on sharding based on namespace or workload alone. As the system increases in complexity, consider adding custom labels related to the application or service to better delineate the targets. -```yaml -scrape_configs: - - job_name: 'otel-collector-dev' - static_configs: - - targets: ['test-service:metrics-port'] - relabel_configs: - - source_labels: ['__meta_kubernetes_namespace'] # Default prometheus label - target_label: 'namespace' - action: keep -``` - -If you have metadata defined for your deployment, you can use that to further -refine the target. - -Example manifest: +Here is an example of how to configure target labels in a collector +configuration that uses a Prometheus receiver. ```yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: test-service - labels: - tier: frontend -``` +# config.yaml example +receivers: + prometheus: + config: + scrape_configs: + - job_name: 'otel-collector-dev-test-service-frontend' + static_configs: + - targets: ['test-service:metrics-port'] + relabel_configs: + - source_labels: ['__meta_kubernetes_namespace'] + target_label: 'namespace' + action: keep + - source_labels: ['__meta_kubernetes_service_label_tier'] + target_label: 'tier' + action: keep + + - job_name: 'otel-collector-dev-test-service-backend' + static_configs: + - targets: ['test-service:metrics-port'] + relabel_configs: + - source_labels: ['__meta_kubernetes_namespace'] + target_label: 'namespace' + action: keep + - source_labels: ['__meta_kubernetes_service_label_tier'] + target_label: 'tier' + action: keep -```yaml -scrape_configs: - - job_name: 'otel-collector-dev-test-service-frontend' - static_configs: - - targets: ['test-service:metrics-port'] - relabel_configs: - - source_labels: ['__meta_kubernetes_namespace'] - target_label: 'namespace' - action: keep - - source_labels: ['__meta_kubernetes_service_label_tier'] - target_label: 'tier' - action: keep - - - job_name: 'otel-collector-dev-test-service-backend' - static_configs: - - targets: ['test-service:metrics-port'] - relabel_configs: - - source_labels: ['__meta_kubernetes_namespace'] - target_label: 'namespace' - action: keep - - source_labels: ['__meta_kubernetes_service_label_tier'] - target_label: 'tier' - action: keep +exporters: + otlp: + endpoint: my.sample:4317 + tls: + insecure: true + +service: + pipelines: + metrics: + receivers: [prometheus] + exporters: [otlp] ``` diff --git a/content/en/docs/concepts/sampling/index.md b/content/en/docs/concepts/sampling/index.md index 4161b680994f..f904ffceea4c 100644 --- a/content/en/docs/concepts/sampling/index.md +++ b/content/en/docs/concepts/sampling/index.md @@ -137,3 +137,11 @@ For the individual language specific implementations of the OpenTelemetry API & SDK you will find support for sampling at the respective documentation pages: {{% sampling-support-list " " %}} + +### Single-Writer Principle + + When scaling collectors for distributed tracing with tail sampling, it's important to consider +the Single-Writer Principle. This principle ensures a single logical writer makes sampling decisions for a +particular trace, avoiding conflicts that might arise from multiple collectors evaluating the same trace for +sampling. Techniques like using load balancing exporters in front of tail-sampling collectors can help achieve this. +