Remove ecs.version field from mappings

We don't use the field at all, and its presence
works against converging on OTel SemConv, and
decoupling the index templates from apm-server.

apmpackage/apm/changelog.yml

@@ -15,6 +15,9 @@
     - description: Explicitly set `event.ingested` in traces-apm.sampled
       type: enhancement
       link: https://github.com/elastic/apm-server/pull/11623
+    - description: Remove `ecs.version` from all data streams
+      type: enhancement
+      link: https://github.com/elastic/apm-server/pull/11632
 - version: "8.10.0"
   changes:
     - description: Add permissions to reroute to dedicated datasets for logs, metrics and traces

apmpackage/apm/data_stream/app_logs/elasticsearch/ingest_pipeline/default.yml

@@ -8,4 +8,4 @@ processors:
   - pipeline:
       name: observer_ids
   - pipeline:
-      name: ecs_version
+      name: remove_ecs_version

apmpackage/apm/data_stream/app_logs/fields/ecs.yml

@@ -70,8 +70,6 @@
   name: device.model.identifier
 - external: ecs
   name: device.manufacturer
-- external: ecs
-  name: ecs.version
 - external: ecs
   name: error.stack_trace
 - external: ecs

apmpackage/apm/data_stream/app_metrics/elasticsearch/ingest_pipeline/default.yml

@@ -6,7 +6,7 @@ processors:
   - pipeline:
       name: observer_ids
   - pipeline:
-      name: ecs_version
+      name: remove_ecs_version
   - pipeline:
       name: user_agent
   - pipeline:

apmpackage/apm/data_stream/app_metrics/fields/ecs.yml

@@ -70,8 +70,6 @@
   name: device.model.identifier
 - external: ecs
   name: device.manufacturer
-- external: ecs
-  name: ecs.version
 - external: ecs
   name: event.outcome
 - external: ecs

apmpackage/apm/data_stream/error_logs/elasticsearch/ingest_pipeline/default.yml

@@ -6,7 +6,7 @@ processors:
   - pipeline:
       name: observer_ids
   - pipeline:
-      name: ecs_version
+      name: remove_ecs_version
   - pipeline:
       name: user_agent
   - pipeline:

apmpackage/apm/data_stream/error_logs/fields/ecs.yml

@@ -70,8 +70,6 @@
   name: device.model.identifier
 - external: ecs
   name: device.manufacturer
-- external: ecs
-  name: ecs.version
 - external: ecs
   name: error.id
 - external: ecs

apmpackage/apm/data_stream/internal_metrics/elasticsearch/ingest_pipeline/default.yml

@@ -6,7 +6,7 @@ processors:
   - pipeline:
       name: observer_ids
   - pipeline:
-      name: ecs_version
+      name: remove_ecs_version
   - pipeline:
       name: user_agent
   - pipeline:

apmpackage/apm/data_stream/internal_metrics/fields/ecs.yml

@@ -62,8 +62,6 @@
   name: destination.ip
 - external: ecs
   name: destination.port
-- external: ecs
-  name: ecs.version
 - external: ecs
   name: event.outcome
 - external: ecs

apmpackage/apm/data_stream/rum_traces/elasticsearch/ingest_pipeline/default.yml

@@ -6,7 +6,7 @@ processors:
   - pipeline:
       name: observer_ids
   - pipeline:
-      name: ecs_version
+      name: remove_ecs_version
   - pipeline:
       name: user_agent
   - pipeline:

apmpackage/apm/data_stream/service_destination_interval_metrics/elasticsearch/ingest_pipeline/default.yml

@@ -6,7 +6,7 @@ processors:
   - pipeline:
       name: observer_ids
   - pipeline:
-      name: ecs_version
+      name: remove_ecs_version
   - pipeline:
       name: set_metrics
   - remove:

apmpackage/apm/data_stream/service_destination_interval_metrics/fields/ecs.yml

@@ -1,7 +1,5 @@
 - external: ecs
   name: agent.name
-- external: ecs
-  name: ecs.version
 - external: ecs
   name: event.outcome
 - external: ecs

apmpackage/apm/data_stream/service_summary_interval_metrics/elasticsearch/ingest_pipeline/default.yml

@@ -6,7 +6,7 @@ processors:
   - pipeline:
       name: observer_ids
   - pipeline:
-      name: ecs_version
+      name: remove_ecs_version
   - pipeline:
       name: set_metrics
   - remove:

apmpackage/apm/data_stream/service_summary_interval_metrics/fields/ecs.yml

@@ -1,7 +1,5 @@
 - external: ecs
   name: agent.name
-- external: ecs
-  name: ecs.version
 - external: ecs
   name: observer.hostname
 - external: ecs

apmpackage/apm/data_stream/service_transaction_interval_metrics/elasticsearch/ingest_pipeline/default.yml

@@ -6,7 +6,7 @@ processors:
   - pipeline:
       name: observer_ids
   - pipeline:
-      name: ecs_version
+      name: remove_ecs_version
   - pipeline:
       name: set_metrics
   - remove:

apmpackage/apm/data_stream/service_transaction_interval_metrics/fields/ecs.yml

@@ -1,7 +1,5 @@
 - external: ecs
   name: agent.name
-- external: ecs
-  name: ecs.version
 - external: ecs
   name: observer.hostname
 - external: ecs

apmpackage/apm/data_stream/traces/elasticsearch/ingest_pipeline/apm_data_stream_migration.yml

@@ -9,7 +9,7 @@ processors:
   - pipeline:
       name: observer_ids
   - pipeline:
-      name: ecs_version
+      name: remove_ecs_version
   - pipeline:
       name: process_ppid
   - script:

apmpackage/apm/data_stream/traces/elasticsearch/ingest_pipeline/default.yml

@@ -6,7 +6,7 @@ processors:
   - pipeline:
       name: observer_ids
   - pipeline:
-      name: ecs_version
+      name: remove_ecs_version
   - pipeline:
       name: user_agent
   - pipeline:

apmpackage/apm/data_stream/traces/fields/ecs.yml

@@ -70,8 +70,6 @@
   name: device.model.identifier
 - external: ecs
   name: device.manufacturer
-- external: ecs
-  name: ecs.version
 - external: ecs
   name: event.outcome
 - external: ecs

apmpackage/apm/data_stream/transaction_interval_metrics/elasticsearch/ingest_pipeline/default.yml

@@ -6,7 +6,7 @@ processors:
   - pipeline:
       name: observer_ids
   - pipeline:
-      name: ecs_version
+      name: remove_ecs_version
   - pipeline:
       name: set_metrics
   - remove:

apmpackage/apm/data_stream/transaction_interval_metrics/fields/ecs.yml

@@ -22,8 +22,6 @@
   name: cloud.service.name
 - external: ecs
   name: container.id
-- external: ecs
-  name: ecs.version
 - external: ecs
   name: event.outcome
 - external: ecs

apmpackage/cmd/genpackage/main.go

@@ -40,7 +40,7 @@ var (
 	ecsVersion   = flag.String("ecs", "", "ECS version (required)")
 )
 
-func generatePackage(pkgfs fs.FS, version, ecsVersion *version.V, ecsReference string) error {
+func generatePackage(pkgfs fs.FS, version *version.V, ecsReference string) error {
 	// Walk files, performing some APM-specific validations and transformations as we go.
 	//
 	// We assume the target destination does not yet exist.
@@ -77,7 +77,7 @@ func generatePackage(pkgfs fs.FS, version, ecsVersion *version.V, ecsReference s
 				// Use `lifecycle.json` instead of e.g. `lifecycle.1m.json`
 				outputPath = strings.Replace(filepath.Join(*outputDir, p.Path), d.Name(), "lifecycle.yml", -1)
 			}
-			err := renderFile(pkgfs, path, outputPath, version, ecsVersion, ecsReference, p.Interval)
+			err := renderFile(pkgfs, path, outputPath, version, ecsReference, p.Interval)
 			if err != nil {
 				return err
 			}
@@ -86,7 +86,7 @@ func generatePackage(pkgfs fs.FS, version, ecsVersion *version.V, ecsReference s
 	})
 }
 
-func renderFile(pkgfs fs.FS, path, outputPath string, version, ecsVersion *version.V, ecsReference, interval string) error {
+func renderFile(pkgfs fs.FS, path, outputPath string, version *version.V, ecsReference, interval string) error {
 	content, err := fs.ReadFile(pkgfs, path)
 	if err != nil {
 		return err
@@ -111,7 +111,7 @@ func renderFile(pkgfs fs.FS, path, outputPath string, version, ecsVersion *versi
 		}
 		content = buf.Bytes()
 	}
-	content, err = transformFile(path, content, version, ecsVersion, ecsReference, interval)
+	content, err = transformFile(path, content, version, ecsReference, interval)
 	if err != nil {
 		return fmt.Errorf("error transforming %q: %w", path, err)
 	}
@@ -176,7 +176,6 @@ func main() {
 		*ecsReference = fmt.Sprintf("git@v%s", *ecsVersion)
 	}
 	pkgVersion := version.MustNew(*pkgVersion)
-	ecsVersion := version.MustNew(*ecsVersion)
 
 	// Locate the apmpackage/apm directory.
 	_, file, _, ok := runtime.Caller(0)
@@ -188,7 +187,7 @@ func main() {
 	// Generate a completely rendered _source_ package, which can then be fed to
 	// `elastic-agent build` to build the final package for inclusion in package-storage.
 	log.Printf("generating integration package v%s in %q", pkgVersion.String(), *outputDir)
-	if err := generatePackage(os.DirFS(pkgdir), pkgVersion, ecsVersion, *ecsReference); err != nil {
+	if err := generatePackage(os.DirFS(pkgdir), pkgVersion, *ecsReference); err != nil {
 		log.Fatal(err)
 	}
 }

apmpackage/cmd/genpackage/pipelines.go

@@ -36,15 +36,15 @@ import (
 //	  - ...
 func getCommonPipeline(name string, version *version.V) []map[string]interface{} {
 	commonPipelines := map[string][]map[string]interface{}{
-		"observer_version": getObserverVersionPipeline(version),
-		"observer_ids":     observerIDsPipeline,
-		"ecs_version":      ecsVersionPipeline,
-		"user_agent":       userAgentPipeline,
-		"process_ppid":     processPpidPipeline,
-		"client_geoip":     clientGeoIPPipeline,
-		"event_duration":   eventDurationPipeline,
-		"set_metrics":      setMetricsPipeline,
-		"event_ingested":   eventIngestedPipeline,
+		"observer_version":   getObserverVersionPipeline(version),
+		"observer_ids":       observerIDsPipeline,
+		"remove_ecs_version": removeECSVersionPipeline,
+		"user_agent":         userAgentPipeline,
+		"process_ppid":       processPpidPipeline,
+		"client_geoip":       clientGeoIPPipeline,
+		"event_duration":     eventDurationPipeline,
+		"set_metrics":        setMetricsPipeline,
+		"event_ingested":     eventIngestedPipeline,
 	}
 	return commonPipelines[name]
 }
@@ -103,7 +103,7 @@ var observerIDsPipeline = []map[string]interface{}{{
 	},
 }}
 
-var ecsVersionPipeline = []map[string]interface{}{{
+var removeECSVersionPipeline = []map[string]interface{}{{
 	"remove": map[string]interface{}{
 		"field":          "ecs", // remove ecs.version
 		"ignore_missing": true,

apmpackage/cmd/genpackage/transform.go

@@ -33,7 +33,7 @@ const (
 	integrationName = "apm"
 )
 
-func transformFile(path string, content []byte, version, ecsVersion *version.V, ecsReference, interval string) ([]byte, error) {
+func transformFile(path string, content []byte, version *version.V, ecsReference, interval string) ([]byte, error) {
 	if path == "manifest.yml" {
 		return transformPackageManifest(content, version)
 	}
@@ -43,9 +43,6 @@ func transformFile(path string, content []byte, version, ecsVersion *version.V,
 	if isIngestPipeline(path) {
 		return transformIngestPipeline(path, content, version)
 	}
-	if isECSFieldsYAML(path) {
-		return transformECSFieldsYAML(content, ecsVersion)
-	}
 	if path == "changelog.yml" {
 		return transformChangelog(content, version)
 	}
@@ -55,33 +52,6 @@ func transformFile(path string, content []byte, version, ecsVersion *version.V,
 	return content, nil
 }
 
-func transformECSFieldsYAML(content []byte, ecsVersion *version.V) ([]byte, error) {
-	var doc yaml.Node
-	if err := yaml.Unmarshal(content, &doc); err != nil {
-		return nil, err
-	}
-
-	var found bool
-	for _, fieldNode := range doc.Content[0].Content {
-		if yamlMapLookup(fieldNode, "name").Value != "ecs.version" {
-			continue
-		}
-		fieldNode.Content = append(fieldNode.Content,
-			&yaml.Node{Kind: yaml.ScalarNode, Value: "type"},
-			&yaml.Node{Kind: yaml.ScalarNode, Value: "constant_keyword"},
-			&yaml.Node{Kind: yaml.ScalarNode, Value: "value"},
-			&yaml.Node{Kind: yaml.ScalarNode, Value: ecsVersion.String()},
-		)
-		found = true
-		break
-	}
-	if !found {
-		return content, nil
-	}
-
-	return marshalYAML(&doc)
-}
-
 func transformBuildDependencies(content []byte, ecsReference string) ([]byte, error) {
 	var doc yaml.Node
 	if err := yaml.Unmarshal(content, &doc); err != nil {
@@ -223,14 +193,6 @@ func isDataStreamManifest(path string) bool {
 	return filepath.Base(dir) == "data_stream"
 }
 
-func isECSFieldsYAML(path string) bool {
-	dir, file := filepath.Split(path)
-	if file != "ecs.yml" {
-		return false
-	}
-	return filepath.Base(dir) == "fields"
-}
-
 func yamlMapLookup(n *yaml.Node, key ...string) *yaml.Node {
 	if n.Kind != yaml.MappingNode {
 		panic(fmt.Sprintf("expected node kind %v, got %v", yaml.MappingNode, n.Kind))

changelogs/head.asciidoc

@@ -5,6 +5,7 @@ https://github.com/elastic/apm-server/compare/8.10\...main[View commits]
 
 [float]
 ==== Breaking Changes
+- The `ecs.version` field has been removed from APM data streams {pull}11632[11632]
 
 [float]
 ==== Deprecations

systemtest/approvals/TestAgentConfig.approved.json

@@ -8,9 +8,6 @@
                 "namespace": "default",
                 "type": "metrics"
             },
-            "ecs": {
-                "version": "dynamic"
-            },
             "labels": {
                 "etag": "dynamic"
             },
@@ -34,9 +31,6 @@
                 "namespace": "default",
                 "type": "metrics"
             },
-            "ecs": {
-                "version": "dynamic"
-            },
             "labels": {
                 "etag": "dynamic"
             },