From 94bb656bc7d981f02dabe79cfc20c21908cc4603 Mon Sep 17 00:00:00 2001 From: Andreas Thaler Date: Mon, 27 Jan 2025 14:38:43 +0100 Subject: [PATCH] feat: Update to OTel-Collector 0.118.0 (#1787) --- .env | 2 +- docs/contributor/benchmarks/README.md | 6 ++++-- internal/images/images.go | 2 +- internal/otelcollector/config/metric/gateway/config.go | 1 - internal/otelcollector/config/metric/gateway/connectors.go | 1 - .../otelcollector/config/metric/gateway/connectors_test.go | 1 - .../config/metric/gateway/testdata/config.yaml | 1 - .../metric/gateway/testdata/config_otlp_disabled.yaml | 1 - sec-scanners-config.yaml | 2 +- 9 files changed, 7 insertions(+), 10 deletions(-) diff --git a/.env b/.env index cd817b514..7f47831d9 100644 --- a/.env +++ b/.env @@ -17,6 +17,6 @@ ENV_GORELEASER_VERSION=v1.23.0 ## Default Docker Images DEFAULT_FLUENTBIT_EXPORTER_IMAGE="europe-docker.pkg.dev/kyma-project/prod/directory-size-exporter:v20241212-e4adf27f" DEFAULT_FLUENTBIT_IMAGE="europe-docker.pkg.dev/kyma-project/prod/external/fluent/fluent-bit:3.2.4" -DEFAULT_OTEL_COLLECTOR_IMAGE="europe-docker.pkg.dev/kyma-project/prod/kyma-otel-collector:0.116.0-main" +DEFAULT_OTEL_COLLECTOR_IMAGE="europe-docker.pkg.dev/kyma-project/prod/kyma-otel-collector:0.118.0-main" DEFAULT_SELFMONITOR_IMAGE="europe-docker.pkg.dev/kyma-project/prod/tpi/telemetry-self-monitor:3.1.0-98bf175" DEFAULT_TEST_TELEMETRYGEN_IMAGE="ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:v0.116.0" \ No newline at end of file diff --git a/docs/contributor/benchmarks/README.md b/docs/contributor/benchmarks/README.md index 65dcbed02..3bba573d5 100644 --- a/docs/contributor/benchmarks/README.md +++ b/docs/contributor/benchmarks/README.md @@ -122,6 +122,7 @@ A typical test result output looks like the following example: | 0.114.0 | 19610 | 19453 | 0 | 127, 125 | 1, 1 | 11256 | 33308 | 0 | 175, 248 | 1.4, 1.4 | 10608 | 321 | 511 | 1737, 1735 | 0.5, 0.5 | 18442 | 956 | 510 | 1798, 1737 | 0.9, 0.9 | | 0.115.0 | 18865 | 18718 | 0 | 191, 253 | 1, 1 | 11615 | 34386 | 0 | 275, 167 | 1.4, 1.5 | 11141 | 277 | 511 | 1747, 1731 | 0.5, 0.5 | 18258 | 880 | 510 | 1741, 1760 | 0.9, 0.9 | | 0.116.0 | 19693 | 19540 | 0 | 165, 126 | 1.1, 1 | 11388 | 33717 | 0 | 196, 137 | 1.5, 1.4 | 11215 | 324 | 510 | 1658, 1738 | 0.5, 0.5 | 17974 | 886 | 509 | 1671, 1683 | 0.9, 0.9 | +| 0.118.0 | 19299 | 19148 | 0 | 88,97, | 1.1,1, | 11369 | 33659 | 0 | 137,159, | 1.4,1.5, | 10066 | 296 | 512 | 1551,1652, | 0.4,0.4, | 18852 | 945 | 510 | 1701,1688, | 0.9,0.9, | @@ -244,7 +245,8 @@ are printed out. | 0.110.0 | 4223 | 4222 | 0 | 130, 137 | 1.5, 1.5 | 3139 | 9417 | 1 | 197, 215 | 1.7, 1.7 | 830 | 640 | 287 | 841, 835 | 0.5, 0.5 | 2048 | 1907 | 510 | 1741, 1694 | 1.4, 1.4 | | 0.114.0 | 4384 | 4385 | 0 | 131, 141 | 1.5, 1.5 | 3209 | 9624 | 0 | 189, 198 | 1.7, 1.8 | 757 | 635 | 393 | 807, 824 | 0.5, 0.4 | 2512 | 1691 | 510 | 1788, 1789 | 1.6, 1.6 | | 0.115.0 | 4256 | 4255 | 0 | 144, 175 | 1.5, 1.5 | 3346 | 10040 | 0 | 244, 202 | 1.7, 1.8 | 726 | 627 | 361 | 821, 834 | 0.5, 0.5 | 2510 | 1926 | 505 | 1778, 1730 | 1.7, 1.6 | -| 0.116.0 | 4374 | 4374 | 0 | 100, 109 | 1.5, 1.5 | 3500 | 10500 | 0 | 171, 171 | 1.8, 2 | 710 | 641 | 383 | 857, 870 | 0.5, 0.5 | 3183 | 1780 | 509 | 1760, 1848 | 2, 2.1 | +| 0.116.0 | 4374 | 4374 | 0 | 100, 109 | 1.5, 1.5 | 3500 | 10500 | 0 | 171, 171 | 1.8, 2 | 710 | 641 | 383 | 857, 870 | 0.5, 0.5 | 3183 | 1780 | 509 | 1760, 1848 | 2, 2.1 | +| 0.118.0 | 4357 | 4357 | 0 | 120,115, | 1.5,1.5, | 3520 | 10566 | 0 | 151,179, | 2,1.8, | 813 | 522 | 443 | 880,1752, | 0.6,0.6, | 3264 | 1925 | 510 | 1837,1855, | 2,2.1, | @@ -294,7 +296,7 @@ On average, memory usage for MetricPipeline instances is ~150MB for a single Pod | 0.114.0 | 19904 | 19904 | 0 | 683, 707 | 0.2, 0.2 | 19942 | 19958 | 0 | 701, 743 | 0.2, 0.2 | | 0.115.0 | 20073 | 20073 | 0 | 697, 697 | 0.2, 0.2 | 19924 | 19954 | 0 | 700, 773 | 0.2, 0.3 | | 0.116.0 | 20058 | 20057 | 0 | 690, 682 | 0.3, 0.3 | 19998 | 19999 | 0 | 713, 692 | 0.2, 0.3 | - +| 0.118.0 | 19859 | 19859 | 0 | 694,672, | 0.2,0.2, | 20057 | 20057 | 0 | 661,664, | 0.2,0.2, | The expected throughput for the MetricPipeline agent receiver is ~20K metrics/sec. Expected memory usage is on average ~700MB, and CPU usage is ~0.2 for each instance. diff --git a/internal/images/images.go b/internal/images/images.go index e91f95c3f..6028515f9 100644 --- a/internal/images/images.go +++ b/internal/images/images.go @@ -6,6 +6,6 @@ package images const ( DefaultFluentBitExporterImage = "europe-docker.pkg.dev/kyma-project/prod/directory-size-exporter:v20241212-e4adf27f" DefaultFluentBitImage = "europe-docker.pkg.dev/kyma-project/prod/external/fluent/fluent-bit:3.2.4" - DefaultOTelCollectorImage = "europe-docker.pkg.dev/kyma-project/prod/kyma-otel-collector:0.116.0-main" + DefaultOTelCollectorImage = "europe-docker.pkg.dev/kyma-project/prod/kyma-otel-collector:0.118.0-main" DefaultSelfMonitorImage = "europe-docker.pkg.dev/kyma-project/prod/tpi/telemetry-self-monitor:3.1.0-98bf175" ) diff --git a/internal/otelcollector/config/metric/gateway/config.go b/internal/otelcollector/config/metric/gateway/config.go index d6d25c56e..40040b910 100644 --- a/internal/otelcollector/config/metric/gateway/config.go +++ b/internal/otelcollector/config/metric/gateway/config.go @@ -97,7 +97,6 @@ type Connectors map[string]any type RoutingConnector struct { DefaultPipelines []string `yaml:"default_pipelines"` ErrorMode string `yaml:"error_mode"` - MatchOnce bool `yaml:"match_once"` Table []RoutingConnectorTableEntry `yaml:"table"` } diff --git a/internal/otelcollector/config/metric/gateway/connectors.go b/internal/otelcollector/config/metric/gateway/connectors.go index 87f94b41d..d25c02361 100644 --- a/internal/otelcollector/config/metric/gateway/connectors.go +++ b/internal/otelcollector/config/metric/gateway/connectors.go @@ -13,7 +13,6 @@ func makeRoutingConnectorConfig(pipelineName string) RoutingConnector { return RoutingConnector{ DefaultPipelines: []string{attributesEnrichmentPipelineID}, ErrorMode: "ignore", - MatchOnce: true, Table: []RoutingConnectorTableEntry{ { Statement: fmt.Sprintf("route() where attributes[\"%s\"] == \"true\"", metric.SkipEnrichmentAttribute), diff --git a/internal/otelcollector/config/metric/gateway/connectors_test.go b/internal/otelcollector/config/metric/gateway/connectors_test.go index d1e4c8d98..6b65e813b 100644 --- a/internal/otelcollector/config/metric/gateway/connectors_test.go +++ b/internal/otelcollector/config/metric/gateway/connectors_test.go @@ -45,7 +45,6 @@ func TestConnectors(t *testing.T) { expectedRoutingConnector := RoutingConnector{ DefaultPipelines: []string{"metrics/test-attributes-enrichment"}, ErrorMode: "ignore", - MatchOnce: true, Table: []RoutingConnectorTableEntry{ { Statement: "route() where attributes[\"io.kyma-project.telemetry.skip_enrichment\"] == \"true\"", diff --git a/internal/otelcollector/config/metric/gateway/testdata/config.yaml b/internal/otelcollector/config/metric/gateway/testdata/config.yaml index 3033c643c..8deeab183 100644 --- a/internal/otelcollector/config/metric/gateway/testdata/config.yaml +++ b/internal/otelcollector/config/metric/gateway/testdata/config.yaml @@ -179,7 +179,6 @@ connectors: default_pipelines: - metrics/test-attributes-enrichment error_mode: ignore - match_once: true table: - statement: route() where attributes["io.kyma-project.telemetry.skip_enrichment"] == "true" pipelines: diff --git a/internal/otelcollector/config/metric/gateway/testdata/config_otlp_disabled.yaml b/internal/otelcollector/config/metric/gateway/testdata/config_otlp_disabled.yaml index de3d7dce5..afbbdcbdd 100644 --- a/internal/otelcollector/config/metric/gateway/testdata/config_otlp_disabled.yaml +++ b/internal/otelcollector/config/metric/gateway/testdata/config_otlp_disabled.yaml @@ -184,7 +184,6 @@ connectors: default_pipelines: - metrics/test-attributes-enrichment error_mode: ignore - match_once: true table: - statement: route() where attributes["io.kyma-project.telemetry.skip_enrichment"] == "true" pipelines: diff --git a/sec-scanners-config.yaml b/sec-scanners-config.yaml index 79c3022c2..48f7eca35 100644 --- a/sec-scanners-config.yaml +++ b/sec-scanners-config.yaml @@ -3,7 +3,7 @@ protecode: - europe-docker.pkg.dev/kyma-project/prod/telemetry-manager:main - europe-docker.pkg.dev/kyma-project/prod/directory-size-exporter:v20241212-e4adf27f - europe-docker.pkg.dev/kyma-project/prod/external/fluent/fluent-bit:3.2.4 -- europe-docker.pkg.dev/kyma-project/prod/kyma-otel-collector:0.116.0-main +- europe-docker.pkg.dev/kyma-project/prod/kyma-otel-collector:0.118.0-main - europe-docker.pkg.dev/kyma-project/prod/tpi/telemetry-self-monitor:3.1.0-98bf175 whitesource: language: golang-mod