Depend on apm-data plugin (#12066)

* docker-compose: enable apm-data

* testing/docker: don't install APM integration

* Update system tests to use apm-data

* Remove wait_for_integration config

* Add changelog

changelogs/head.asciidoc

@@ -7,6 +7,7 @@ https://github.com/elastic/apm-server/compare/8.14\...main[View commits]
 ==== Bug fixes
 
 - Avoid data race due to reuse of `bytes.Buffer` in ES bulk requests {pull}13155[13155]
+- APM Server now relies on the Elasticsearch apm-data plugin's index templates, which reverts some unsafe uses of `flattened` field types {pull}12066[12066]
 
 [float]
 ==== Breaking Changes
@@ -21,3 +22,4 @@ https://github.com/elastic/apm-server/compare/8.14\...main[View commits]
 ==== Added
 
 - Add `elasticsearch.flushed.uncompressed.bytes` metric {pull}13155[13155]
+- APM Server now relies on the Elasticsearch apm-data plugin's index templates, removing the requirement to install the APM integration package {pull}12066[12066]

docker-compose.yml

@@ -24,6 +24,7 @@ services:
       - "http.host=0.0.0.0"
       - "cluster.routing.allocation.disk.threshold_enabled=false"
       - "discovery.type=single-node"
+      - "xpack.apm_data.enabled=true"
       - "xpack.security.authc.anonymous.roles=remote_monitoring_collector"
       - "xpack.security.authc.realms.file.file1.order=0"
       - "xpack.security.authc.realms.native.native1.order=1"

internal/beater/beater.go

@@ -85,7 +85,6 @@ type Runner struct {
 	rawConfig  *agentconfig.C
 
 	config                    *config.Config
-	fleetConfig               *config.Fleet
 	outputConfig              agentconfig.Namespace
 	elasticsearchOutputConfig *agentconfig.C
 
@@ -113,7 +112,6 @@ func NewRunner(args RunnerParams) (*Runner, error) {
 	var unpackedConfig struct {
 		APMServer  *agentconfig.C        `config:"apm-server"`
 		Output     agentconfig.Namespace `config:"output"`
-		Fleet      *config.Fleet         `config:"fleet"`
 		DataStream struct {
 			Namespace string `config:"namespace"`
 		} `config:"data_stream"`
@@ -147,7 +145,6 @@ func NewRunner(args RunnerParams) (*Runner, error) {
 		rawConfig:  args.Config,
 
 		config:                    cfg,
-		fleetConfig:               unpackedConfig.Fleet,
 		outputConfig:              unpackedConfig.Output,
 		elasticsearchOutputConfig: elasticsearchOutputConfig,
 
@@ -328,7 +325,7 @@ func (s *Runner) Run(ctx context.Context) error {
 	publishReady := make(chan struct{})
 	drain := make(chan struct{})
 	g.Go(func() error {
-		if err := s.waitReady(ctx, kibanaClient, tracer); err != nil {
+		if err := s.waitReady(ctx, tracer); err != nil {
 			// One or more preconditions failed; drop events.
 			close(drain)
 			return errors.Wrap(err, "error waiting for server to be ready")
@@ -567,7 +564,6 @@ func linearScaledValue(perGBIncrement, memLimitGB, constant float64) int {
 // waitReady waits until the server is ready to index events.
 func (s *Runner) waitReady(
 	ctx context.Context,
-	kibanaClient *kibana.Client,
 	tracer *apm.Tracer,
 ) error {
 	var preconditions []func(context.Context) error
@@ -618,18 +614,6 @@ func (s *Runner) waitReady(
 		})
 	}
 
-	// When running standalone with data streams enabled, by default we will add
-	// a precondition that ensures the integration is installed.
-	fleetManaged := s.fleetConfig != nil
-	if !fleetManaged && s.config.DataStreams.WaitForIntegration {
-		if kibanaClient == nil && esOutputClient == nil {
-			return errors.New("cannot wait for integration without either Kibana or Elasticsearch config")
-		}
-		preconditions = append(preconditions, func(ctx context.Context) error {
-			return checkIntegrationInstalled(ctx, kibanaClient, esOutputClient, s.logger)
-		})
-	}
-
 	if len(preconditions) == 0 {
 		return nil
 	}

internal/beater/beatertest/server.go

@@ -86,10 +86,6 @@ func NewUnstartedServer(t testing.TB, opts ...option) *Server {
 		config: []*agentconfig.C{agentconfig.MustNewConfigFrom(map[string]interface{}{
 			"apm-server": map[string]interface{}{
 				"host": "localhost:0",
-
-				// Disable waiting for integration to be installed by default,
-				// to simplify tests. This feature is tested independently.
-				"data_streams.wait_for_integration": false,
 			},
 		})},
 	}
@@ -206,7 +202,6 @@ type option func(*options)
 //
 //	apm-server:
 //	  host: localhost:0
-//	  data_streams.wait_for_integration: false
 func WithConfig(cfg ...*agentconfig.C) option {
 	return func(opts *options) {
 		opts.config = append(opts.config, cfg...)

internal/beater/config/config_test.go

@@ -370,8 +370,7 @@ func TestUnpackConfig(t *testing.T) {
 				},
 				DefaultServiceEnvironment: "overridden",
 				DataStreams: DataStreamsConfig{
-					Namespace:          "default",
-					WaitForIntegration: true,
+					Namespace: "default",
 				},
 				WaitReadyInterval: 5 * time.Second,
 			},
@@ -419,8 +418,7 @@ func TestUnpackConfig(t *testing.T) {
 					"storage_limit":     "1GB",
 				},
 				"data_streams": map[string]interface{}{
-					"namespace":            "foo",
-					"wait_for_integration": false,
+					"namespace": "foo",
 				},
 			},
 			outCfg: &Config{
@@ -503,8 +501,7 @@ func TestUnpackConfig(t *testing.T) {
 					},
 				},
 				DataStreams: DataStreamsConfig{
-					Namespace:          "foo",
-					WaitForIntegration: false,
+					Namespace: "foo",
 				},
 				WaitReadyInterval: 5 * time.Second,
 			},

internal/beater/config/data_streams.go

@@ -20,21 +20,10 @@ package config
 // DataStreamsConfig holds data streams configuration.
 type DataStreamsConfig struct {
 	Namespace string `config:"namespace"`
-
-	// WaitForIntegration controls whether APM Server waits for the Fleet
-	// integration package to be installed before indexing events.
-	//
-	// This config is ignored when running under Elastic Agent; it is intended
-	// for running APM Server standalone, relying on Fleet to install the integration
-	// for creating Elasticsearch index templates, ILM policies, and ingest pipelines.
-	//
-	// This configuration requires either a connection to Kibana or Elasticsearch.
-	WaitForIntegration bool `config:"wait_for_integration"`
 }
 
 func defaultDataStreamsConfig() DataStreamsConfig {
 	return DataStreamsConfig{
-		Namespace:          "default",
-		WaitForIntegration: true,
+		Namespace: "default",
 	}
 }