[exporter/elasticsearch] Support preserving attributes when mapping to ECS

.chloggen/es_preserve_host_name.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: elasticsearchexporter
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Preserve `host.name` resource attribute in ECS mode
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [33670]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

exporter/elasticsearchexporter/model.go

@@ -49,6 +49,13 @@ var resourceAttrsConversionMap = map[string]string{
 	"k8s.pod.uid":                           "kubernetes.pod.uid",
 }
 
+// resourceAttrsToPreserve contains conventions that should be preserved in ECS mode.
+// This can happen when an attribute needs to be mapped to an ECS equivalent but
+// at the same time be preserved to its original form.
+var resourceAttrsToPreserve = map[string]bool{
+	semconv.AttributeHostName: true,
+}
+
 type mappingModel interface {
 	encodeLog(pcommon.Resource, plog.LogRecord, pcommon.InstrumentationScope) ([]byte, error)
 	encodeSpan(pcommon.Resource, ptrace.Span, pcommon.InstrumentationScope) ([]byte, error)
@@ -117,13 +124,13 @@ func (m *encodeModel) encodeLogECSMode(resource pcommon.Resource, record plog.Lo
 	var document objmodel.Document
 
 	// First, try to map resource-level attributes to ECS fields.
-	encodeLogAttributesECSMode(&document, resource.Attributes(), resourceAttrsConversionMap)
+	encodeLogAttributesECSMode(&document, resource.Attributes(), resourceAttrsConversionMap, resourceAttrsToPreserve)
 
 	// Then, try to map scope-level attributes to ECS fields.
 	scopeAttrsConversionMap := map[string]string{
 		// None at the moment
 	}
-	encodeLogAttributesECSMode(&document, scope.Attributes(), scopeAttrsConversionMap)
+	encodeLogAttributesECSMode(&document, scope.Attributes(), scopeAttrsConversionMap, resourceAttrsToPreserve)
 
 	// Finally, try to map record-level attributes to ECS fields.
 	recordAttrsConversionMap := map[string]string{
@@ -133,7 +140,7 @@ func (m *encodeModel) encodeLogECSMode(resource pcommon.Resource, record plog.Lo
 		semconv.AttributeExceptionType:       "error.type",
 		semconv.AttributeExceptionEscaped:    "event.error.exception.handled",
 	}
-	encodeLogAttributesECSMode(&document, record.Attributes(), recordAttrsConversionMap)
+	encodeLogAttributesECSMode(&document, record.Attributes(), recordAttrsConversionMap, resourceAttrsToPreserve)
 
 	// Handle special cases.
 	encodeLogAgentNameECSMode(&document, resource)
@@ -230,7 +237,7 @@ func scopeToAttributes(scope pcommon.InstrumentationScope) pcommon.Map {
 	return attrs
 }
 
-func encodeLogAttributesECSMode(document *objmodel.Document, attrs pcommon.Map, conversionMap map[string]string) {
+func encodeLogAttributesECSMode(document *objmodel.Document, attrs pcommon.Map, conversionMap map[string]string, preserveMap map[string]bool) {
 	if len(conversionMap) == 0 {
 		// No conversions to be done; add all attributes at top level of
 		// document.
@@ -247,6 +254,9 @@ func encodeLogAttributesECSMode(document *objmodel.Document, attrs pcommon.Map,
 			}
 
 			document.AddAttribute(ecsKey, v)
+			if preserve := preserveMap[k]; preserve {
+				document.AddAttribute(k, v)
+			}
 			return true
 		}
 

exporter/elasticsearchexporter/model_test.go

@@ -231,7 +231,7 @@ func TestEncodeLogECSModeDuplication(t *testing.T) {
 	})
 	require.NoError(t, err)
 
-	want := `{"@timestamp":"2024-03-12T20:00:41.123456789Z","agent":{"name":"otlp"},"container":{"image":{"tag":["v3.4.0"]}},"event":{"action":"user-password-change"},"host":{"hostname":"localhost","os":{"full":"Mac OS Mojave","name":"Mac OS X","platform":"darwin","type":"macos","version":"10.14.1"}},"service":{"name":"foo.bar","version":"1.1.0"}}`
+	want := `{"@timestamp":"2024-03-12T20:00:41.123456789Z","agent":{"name":"otlp"},"container":{"image":{"tag":["v3.4.0"]}},"event":{"action":"user-password-change"},"host":{"hostname":"localhost","name":"localhost","os":{"full":"Mac OS Mojave","name":"Mac OS X","platform":"darwin","type":"macos","version":"10.14.1"}},"service":{"name":"foo.bar","version":"1.1.0"}}`
 	require.NoError(t, err)
 
 	resourceContainerImageTags := resource.Attributes().PutEmptySlice(semconv.AttributeContainerImageTags)
@@ -336,6 +336,7 @@ func TestEncodeLogECSMode(t *testing.T) {
 		"container.image.name":    "my-app",
 		"container.runtime":       "docker",
 		"host.hostname":           "i-103de39e0a.gke.us-west-1b.cloud.google.com",
+		"host.name":               "i-103de39e0a.gke.us-west-1b.cloud.google.com",
 		"host.id":                 "i-103de39e0a",
 		"host.type":               "t2.medium",
 		"host.architecture":       "x86_64",
@@ -671,6 +672,7 @@ func TestMapLogAttributesToECS(t *testing.T) {
 	tests := map[string]struct {
 		attrs         func() pcommon.Map
 		conversionMap map[string]string
+		preserveMap   map[string]bool
 		expectedDoc   func() objmodel.Document
 	}{
 		"no_attrs": {
@@ -775,12 +777,31 @@ func TestMapLogAttributesToECS(t *testing.T) {
 				return d
 			},
 		},
+		"preserve_map": {
+			attrs: func() pcommon.Map {
+				m := pcommon.NewMap()
+				m.PutStr("foo.bar", "baz")
+				return m
+			},
+			conversionMap: map[string]string{
+				"foo.bar": "bar.qux",
+				"qux":     "foo",
+			}, preserveMap: map[string]bool{
+				"foo.bar": true,
+			},
+			expectedDoc: func() objmodel.Document {
+				d := objmodel.Document{}
+				d.AddString("bar.qux", "baz")
+				d.AddString("foo.bar", "baz")
+				return d
+			},
+		},
 	}
 
 	for name, test := range tests {
 		t.Run(name, func(t *testing.T) {
 			var doc objmodel.Document
-			encodeLogAttributesECSMode(&doc, test.attrs(), test.conversionMap)
+			encodeLogAttributesECSMode(&doc, test.attrs(), test.conversionMap, test.preserveMap)
 
 			doc.Sort()
 			expectedDoc := test.expectedDoc()