diff --git a/.chloggen/es_preserve_host_name.yaml b/.chloggen/es_preserve_host_name.yaml new file mode 100644 index 0000000000000..a59ea752cf396 --- /dev/null +++ b/.chloggen/es_preserve_host_name.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: elasticsearchexporter + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Preserve `host.name` resource attribute in ECS mode + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [33670] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] diff --git a/exporter/elasticsearchexporter/model.go b/exporter/elasticsearchexporter/model.go index 1bb8fd2c4b437..36702776e791c 100644 --- a/exporter/elasticsearchexporter/model.go +++ b/exporter/elasticsearchexporter/model.go @@ -49,6 +49,13 @@ var resourceAttrsConversionMap = map[string]string{ "k8s.pod.uid": "kubernetes.pod.uid", } +// resourceAttrsToPreserve contains conventions that should be preserved in ECS mode. +// This can happen when an attribute needs to be mapped to an ECS equivalent but +// at the same time be preserved to its original form. +var resourceAttrsToPreserve = map[string]bool{ + semconv.AttributeHostName: true, +} + type mappingModel interface { encodeLog(pcommon.Resource, plog.LogRecord, pcommon.InstrumentationScope) ([]byte, error) encodeSpan(pcommon.Resource, ptrace.Span, pcommon.InstrumentationScope) ([]byte, error) @@ -117,13 +124,13 @@ func (m *encodeModel) encodeLogECSMode(resource pcommon.Resource, record plog.Lo var document objmodel.Document // First, try to map resource-level attributes to ECS fields. - encodeLogAttributesECSMode(&document, resource.Attributes(), resourceAttrsConversionMap) + encodeLogAttributesECSMode(&document, resource.Attributes(), resourceAttrsConversionMap, resourceAttrsToPreserve) // Then, try to map scope-level attributes to ECS fields. scopeAttrsConversionMap := map[string]string{ // None at the moment } - encodeLogAttributesECSMode(&document, scope.Attributes(), scopeAttrsConversionMap) + encodeLogAttributesECSMode(&document, scope.Attributes(), scopeAttrsConversionMap, resourceAttrsToPreserve) // Finally, try to map record-level attributes to ECS fields. recordAttrsConversionMap := map[string]string{ @@ -133,7 +140,7 @@ func (m *encodeModel) encodeLogECSMode(resource pcommon.Resource, record plog.Lo semconv.AttributeExceptionType: "error.type", semconv.AttributeExceptionEscaped: "event.error.exception.handled", } - encodeLogAttributesECSMode(&document, record.Attributes(), recordAttrsConversionMap) + encodeLogAttributesECSMode(&document, record.Attributes(), recordAttrsConversionMap, resourceAttrsToPreserve) // Handle special cases. encodeLogAgentNameECSMode(&document, resource) @@ -230,7 +237,7 @@ func scopeToAttributes(scope pcommon.InstrumentationScope) pcommon.Map { return attrs } -func encodeLogAttributesECSMode(document *objmodel.Document, attrs pcommon.Map, conversionMap map[string]string) { +func encodeLogAttributesECSMode(document *objmodel.Document, attrs pcommon.Map, conversionMap map[string]string, preserveMap map[string]bool) { if len(conversionMap) == 0 { // No conversions to be done; add all attributes at top level of // document. @@ -247,6 +254,9 @@ func encodeLogAttributesECSMode(document *objmodel.Document, attrs pcommon.Map, } document.AddAttribute(ecsKey, v) + if preserve := preserveMap[k]; preserve { + document.AddAttribute(k, v) + } return true } diff --git a/exporter/elasticsearchexporter/model_test.go b/exporter/elasticsearchexporter/model_test.go index a8c652a64485e..5fb15b05f6faa 100644 --- a/exporter/elasticsearchexporter/model_test.go +++ b/exporter/elasticsearchexporter/model_test.go @@ -231,7 +231,7 @@ func TestEncodeLogECSModeDuplication(t *testing.T) { }) require.NoError(t, err) - want := `{"@timestamp":"2024-03-12T20:00:41.123456789Z","agent":{"name":"otlp"},"container":{"image":{"tag":["v3.4.0"]}},"event":{"action":"user-password-change"},"host":{"hostname":"localhost","os":{"full":"Mac OS Mojave","name":"Mac OS X","platform":"darwin","type":"macos","version":"10.14.1"}},"service":{"name":"foo.bar","version":"1.1.0"}}` + want := `{"@timestamp":"2024-03-12T20:00:41.123456789Z","agent":{"name":"otlp"},"container":{"image":{"tag":["v3.4.0"]}},"event":{"action":"user-password-change"},"host":{"hostname":"localhost","name":"localhost","os":{"full":"Mac OS Mojave","name":"Mac OS X","platform":"darwin","type":"macos","version":"10.14.1"}},"service":{"name":"foo.bar","version":"1.1.0"}}` require.NoError(t, err) resourceContainerImageTags := resource.Attributes().PutEmptySlice(semconv.AttributeContainerImageTags) @@ -336,6 +336,7 @@ func TestEncodeLogECSMode(t *testing.T) { "container.image.name": "my-app", "container.runtime": "docker", "host.hostname": "i-103de39e0a.gke.us-west-1b.cloud.google.com", + "host.name": "i-103de39e0a.gke.us-west-1b.cloud.google.com", "host.id": "i-103de39e0a", "host.type": "t2.medium", "host.architecture": "x86_64", @@ -671,6 +672,7 @@ func TestMapLogAttributesToECS(t *testing.T) { tests := map[string]struct { attrs func() pcommon.Map conversionMap map[string]string + preserveMap map[string]bool expectedDoc func() objmodel.Document }{ "no_attrs": { @@ -775,12 +777,31 @@ func TestMapLogAttributesToECS(t *testing.T) { return d }, }, + "preserve_map": { + attrs: func() pcommon.Map { + m := pcommon.NewMap() + m.PutStr("foo.bar", "baz") + return m + }, + conversionMap: map[string]string{ + "foo.bar": "bar.qux", + "qux": "foo", + }, preserveMap: map[string]bool{ + "foo.bar": true, + }, + expectedDoc: func() objmodel.Document { + d := objmodel.Document{} + d.AddString("bar.qux", "baz") + d.AddString("foo.bar", "baz") + return d + }, + }, } for name, test := range tests { t.Run(name, func(t *testing.T) { var doc objmodel.Document - encodeLogAttributesECSMode(&doc, test.attrs(), test.conversionMap) + encodeLogAttributesECSMode(&doc, test.attrs(), test.conversionMap, test.preserveMap) doc.Sort() expectedDoc := test.expectedDoc()