From ebfd2f9a091562a8456b622a81f91c790208db59 Mon Sep 17 00:00:00 2001 From: dmathieu <42@dmathieu.com> Date: Wed, 8 Jan 2025 16:11:08 +0100 Subject: [PATCH 1/5] [exporters/elasticsearch] remove stale deprecated index option --- exporter/elasticsearchexporter/README.md | 7 ++----- exporter/elasticsearchexporter/config.go | 8 -------- exporter/elasticsearchexporter/config_test.go | 11 ----------- exporter/elasticsearchexporter/factory.go | 8 +------- exporter/elasticsearchexporter/factory_test.go | 18 ------------------ 5 files changed, 3 insertions(+), 49 deletions(-) diff --git a/exporter/elasticsearchexporter/README.md b/exporter/elasticsearchexporter/README.md index 13ecfa53507d..5697db15c4a2 100644 --- a/exporter/elasticsearchexporter/README.md +++ b/exporter/elasticsearchexporter/README.md @@ -115,9 +115,6 @@ Telemetry data will be written to signal specific data streams by default: logs to `logs-generic-default`, metrics to `metrics-generic-default`, and traces to `traces-generic-default`. This can be customised through the following settings: -- `index` (DEPRECATED, please use `logs_index` for logs, `metrics_index` for metrics, `traces_index` for traces): The [index] or [data stream] name to publish events to. - The default value is `logs-generic-default`. - - `logs_index`: The [index] or [data stream] name to publish events to. The default value is `logs-generic-default` - `logs_dynamic_index` (optional): uses resource, scope, or log record attributes to dynamically construct index name. @@ -161,7 +158,7 @@ behaviours, which may be configured through the following settings: - `data_stream.dataset` will always be appended with `.otel`. It is recommended to use with `*_dynamic_index.enabled: true` to route documents to data stream `${data_stream.type}-${data_stream.dataset}-${data_stream.namespace}`. - Span events are stored in separate documents. They will be routed with `data_stream.type` set to `logs` if `traces_dynamic_index::enabled` is `true`. - - `raw`: Omit the `Attributes.` string prefixed to field names for log and + - `raw`: Omit the `Attributes.` string prefixed to field names for log and span attributes as well as omit the `Events.` string prefixed to field names for span events. - `bodymap`: Provides fine-grained control over the final documents to be ingested. @@ -361,4 +358,4 @@ When sending high traffic of metrics to a TSDB metrics data stream, e.g. using O This will be fixed in a future version of Elasticsearch. A possible workaround would be to use a transform processor to truncate the timestamp, but this will cause duplicate data to be dropped silently. -However, if `@timestamp` precision is not the problem, check your metrics pipeline setup for misconfiguration that causes an actual violation of the [single writer principle](https://opentelemetry.io/docs/specs/otel/metrics/data-model/#single-writer). \ No newline at end of file +However, if `@timestamp` precision is not the problem, check your metrics pipeline setup for misconfiguration that causes an actual violation of the [single writer principle](https://opentelemetry.io/docs/specs/otel/metrics/data-model/#single-writer). diff --git a/exporter/elasticsearchexporter/config.go b/exporter/elasticsearchexporter/config.go index 0835396d928f..12dc5651204a 100644 --- a/exporter/elasticsearchexporter/config.go +++ b/exporter/elasticsearchexporter/config.go @@ -38,14 +38,6 @@ type Config struct { // NumWorkers configures the number of workers publishing bulk requests. NumWorkers int `mapstructure:"num_workers"` - // Index configures the index, index alias, or data stream name events should be indexed in. - // - // https://www.elastic.co/guide/en/elasticsearch/reference/current/indices.html - // https://www.elastic.co/guide/en/elasticsearch/reference/current/data-streams.html - // - // Deprecated: `index` is deprecated and replaced with `logs_index`. - Index string `mapstructure:"index"` - // This setting is required when logging pipelines used. LogsIndex string `mapstructure:"logs_index"` // fall back to pure LogsIndex, if 'elasticsearch.index.prefix' or 'elasticsearch.index.suffix' are not found in resource or attribute (prio: resource > attribute) diff --git a/exporter/elasticsearchexporter/config_test.go b/exporter/elasticsearchexporter/config_test.go index b83beb3e91ba..153001b149e2 100644 --- a/exporter/elasticsearchexporter/config_test.go +++ b/exporter/elasticsearchexporter/config_test.go @@ -61,7 +61,6 @@ func TestConfig(t *testing.T) { QueueSize: exporterhelper.NewDefaultQueueConfig().QueueSize, }, Endpoints: []string{"https://elastic.example.com:9200"}, - Index: "", LogsIndex: "logs-generic-default", LogsDynamicIndex: DynamicIndexSetting{ Enabled: false, @@ -133,7 +132,6 @@ func TestConfig(t *testing.T) { QueueSize: exporterhelper.NewDefaultQueueConfig().QueueSize, }, Endpoints: []string{"http://localhost:9200"}, - Index: "", LogsIndex: "my_log_index", LogsDynamicIndex: DynamicIndexSetting{ Enabled: false, @@ -205,7 +203,6 @@ func TestConfig(t *testing.T) { QueueSize: exporterhelper.NewDefaultQueueConfig().QueueSize, }, Endpoints: []string{"http://localhost:9200"}, - Index: "", LogsIndex: "logs-generic-default", LogsDynamicIndex: DynamicIndexSetting{ Enabled: false, @@ -284,14 +281,6 @@ func TestConfig(t *testing.T) { cfg.CloudID = "foo:YmFyLmNsb3VkLmVzLmlvJGFiYzEyMyRkZWY0NTY=" }), }, - { - id: component.NewIDWithName(metadata.Type, "deprecated_index"), - configFile: "config.yaml", - expected: withDefaultConfig(func(cfg *Config) { - cfg.Endpoints = []string{"https://elastic.example.com:9200"} - cfg.Index = "my_log_index" - }), - }, { id: component.NewIDWithName(metadata.Type, "confighttp_endpoint"), configFile: "config.yaml", diff --git a/exporter/elasticsearchexporter/factory.go b/exporter/elasticsearchexporter/factory.go index 4783d430196a..887e00cc63ed 100644 --- a/exporter/elasticsearchexporter/factory.go +++ b/exporter/elasticsearchexporter/factory.go @@ -50,7 +50,6 @@ func createDefaultConfig() component.Config { return &Config{ QueueSettings: qs, ClientConfig: httpClientConfig, - Index: "", LogsIndex: defaultLogsIndex, LogsDynamicIndex: DynamicIndexSetting{ Enabled: false, @@ -111,14 +110,9 @@ func createLogsExporter( ) (exporter.Logs, error) { cf := cfg.(*Config) - index := cf.LogsIndex - if cf.Index != "" { - set.Logger.Warn("index option are deprecated and replaced with logs_index and traces_index.") - index = cf.Index - } handleDeprecatedConfig(cf, set.Logger) - exporter := newExporter(cf, set, index, cf.LogsDynamicIndex.Enabled) + exporter := newExporter(cf, set, cf.LogsIndex, cf.LogsDynamicIndex.Enabled) return exporterhelper.NewLogs( ctx, diff --git a/exporter/elasticsearchexporter/factory_test.go b/exporter/elasticsearchexporter/factory_test.go index e24dca1af5e6..80f294f37705 100644 --- a/exporter/elasticsearchexporter/factory_test.go +++ b/exporter/elasticsearchexporter/factory_test.go @@ -61,24 +61,6 @@ func TestFactory_CreateTraces(t *testing.T) { require.NoError(t, exporter.Shutdown(context.Background())) } -func TestFactory_CreateLogsAndTracesExporterWithDeprecatedIndexOption(t *testing.T) { - factory := NewFactory() - cfg := withDefaultConfig(func(cfg *Config) { - cfg.Endpoints = []string{"http://test:9200"} - cfg.Index = "test_index" - }) - params := exportertest.NewNopSettings() - logsExporter, err := factory.CreateLogs(context.Background(), params, cfg) - require.NoError(t, err) - require.NotNil(t, logsExporter) - require.NoError(t, logsExporter.Shutdown(context.Background())) - - tracesExporter, err := factory.CreateTraces(context.Background(), params, cfg) - require.NoError(t, err) - require.NotNil(t, tracesExporter) - require.NoError(t, tracesExporter.Shutdown(context.Background())) -} - func TestFactory_DedupDeprecated(t *testing.T) { factory := NewFactory() cfg := withDefaultConfig(func(cfg *Config) { From 5dab9579a20a29426e1942457ee606e0374bf75b Mon Sep 17 00:00:00 2001 From: dmathieu <42@dmathieu.com> Date: Wed, 8 Jan 2025 16:12:29 +0100 Subject: [PATCH 2/5] add changelog entry --- .chloggen/elasticsearch-remove-index.yaml | 28 +++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .chloggen/elasticsearch-remove-index.yaml diff --git a/.chloggen/elasticsearch-remove-index.yaml b/.chloggen/elasticsearch-remove-index.yaml new file mode 100644 index 000000000000..d31b2601c858 --- /dev/null +++ b/.chloggen/elasticsearch-remove-index.yaml @@ -0,0 +1,28 @@ + +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: breaking + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: exporter/elasticsearch + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Remove Index config, which was deprecated since v0.60.0 + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [37094] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] From 2d3770e227c0be1a293ebfc7999ccc00b83f4437 Mon Sep 17 00:00:00 2001 From: Damien Mathieu <42@dmathieu.com> Date: Wed, 8 Jan 2025 16:30:29 +0100 Subject: [PATCH 3/5] Update .chloggen/elasticsearch-remove-index.yaml Co-authored-by: Carson Ip --- .chloggen/elasticsearch-remove-index.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.chloggen/elasticsearch-remove-index.yaml b/.chloggen/elasticsearch-remove-index.yaml index d31b2601c858..05947591298b 100644 --- a/.chloggen/elasticsearch-remove-index.yaml +++ b/.chloggen/elasticsearch-remove-index.yaml @@ -5,7 +5,7 @@ change_type: breaking # The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) -component: exporter/elasticsearch +component: elasticsearchexporter # A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). note: Remove Index config, which was deprecated since v0.60.0 From 883e4968938ec7301c8dff115675bf92dbfbe087 Mon Sep 17 00:00:00 2001 From: dmathieu <42@dmathieu.com> Date: Thu, 9 Jan 2025 09:31:02 +0100 Subject: [PATCH 4/5] specify replacement config --- .chloggen/elasticsearch-remove-index.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.chloggen/elasticsearch-remove-index.yaml b/.chloggen/elasticsearch-remove-index.yaml index 05947591298b..e60812494396 100644 --- a/.chloggen/elasticsearch-remove-index.yaml +++ b/.chloggen/elasticsearch-remove-index.yaml @@ -8,7 +8,7 @@ change_type: breaking component: elasticsearchexporter # A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). -note: Remove Index config, which was deprecated since v0.60.0 +note: Remove Index config, which was deprecated and replace with LogsIndex since v0.60.0 # Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. issues: [37094] From 74475f274c65a460d5b4aac2f3ffc0f936bcce9b Mon Sep 17 00:00:00 2001 From: dmathieu <42@dmathieu.com> Date: Thu, 9 Jan 2025 09:31:37 +0100 Subject: [PATCH 5/5] remove deprecated index from test data --- exporter/elasticsearchexporter/testdata/config.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/exporter/elasticsearchexporter/testdata/config.yaml b/exporter/elasticsearchexporter/testdata/config.yaml index e3f7ffc67fa9..92945c7f1b1c 100644 --- a/exporter/elasticsearchexporter/testdata/config.yaml +++ b/exporter/elasticsearchexporter/testdata/config.yaml @@ -77,9 +77,6 @@ elasticsearch/raw: mode: raw elasticsearch/cloudid: cloudid: foo:YmFyLmNsb3VkLmVzLmlvJGFiYzEyMyRkZWY0NTY= -elasticsearch/deprecated_index: - endpoints: [https://elastic.example.com:9200] - index: my_log_index elasticsearch/confighttp_endpoint: endpoint: https://elastic.example.com:9200 elasticsearch/batcher_disabled: