Expose local metrics to the batch processor (#11582)

* expose request/response counters to local exporter

* add changelog entry

* add PR number

* add local collection doc

changelogs/head.asciidoc

@@ -20,3 +20,14 @@ https://github.com/elastic/apm-server/compare/8.10\...main[View commits]
 [float]
 ==== Added
 - Support and define DLM data retention period in the apmpackage
+- Expose new metrics into the local batch processor {pull}11582[11582] :
+	- http.server.request.count
+	- http.server.response.valid.count
+	- http.server.response.errors.count
+	- http.server.errors.timeout
+	- http.server.errors.ratelimit
+	- grpc.server.request.count
+	- grpc.server.response.valid.count
+	- grpc.server.response.errors.count
+	- grpc.server.errors.timeout
+	- grpc.server.errors.ratelimit

docs/monitoring/monitoring-beats.asciidoc

@@ -15,9 +15,13 @@ ifndef::serverless[]
 * <<monitoring-metricbeat-collection, {metricbeat} collection>> -
 {metricbeat} collects monitoring data from your {beatname_uc} instance
 and sends it directly to your monitoring cluster.
+* <<monitoring-local-collection, Local collection>> - Local collection sends
+select monitoring data directly to the standard indices of your monitoring
+cluster.
 endif::[]
 
 include::monitoring-internal-collection.asciidoc[]
+include::monitoring-local-collection.asciidoc[]
 
 ifndef::serverless[]
 include::monitoring-metricbeat.asciidoc[]

docs/monitoring/monitoring-local-collection.asciidoc

@@ -0,0 +1,32 @@
+[[monitoring-local-collection]]
+= Use the select metrics emitted directly to your monitoring cluster
+++++
+<titleabbrev>Use local collection</titleabbrev>
+++++
+
+In 8.10 and later, we emit a selected set of metrics directly to the monitoring
+cluster.
+The benefit of using local collection instead of internal collection is that
+the metrics are sent directly to your main monitoring index, making it easier
+to view shared data.
+
+[[select-metrics]]
+== The select metrics
+
+We only ship a select list of metrics, to avoid overwhelming your monitoring cluster.
+If you need the entire set of metrics and traces we can expose, you should use
+<<configuration-instrumentation,Self Instrumentation>> instead of local
+collection.
+
+Here is the list of every metrics we currently expose:
+
+* http.server.request.count
+* http.server.response.valid.count
+* http.server.response.errors.count
+* http.server.errors.timeout
+* http.server.errors.ratelimit
+* grpc.server.request.count
+* grpc.server.response.valid.count
+* grpc.server.response.errors.count
+* grpc.server.errors.timeout
+* grpc.server.errors.ratelimit

internal/beater/beater.go

@@ -63,6 +63,7 @@ import (
 	"github.com/elastic/apm-server/internal/beater/interceptors"
 	javaattacher "github.com/elastic/apm-server/internal/beater/java_attacher"
 	"github.com/elastic/apm-server/internal/beater/ratelimit"
+	"github.com/elastic/apm-server/internal/beater/request"
 	"github.com/elastic/apm-server/internal/elasticsearch"
 	"github.com/elastic/apm-server/internal/idxmgmt"
 	"github.com/elastic/apm-server/internal/kibana"
@@ -311,7 +312,21 @@ func (s *Runner) Run(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
-	localExporter := telemetry.NewMetricExporter()
+	localExporter := telemetry.NewMetricExporter(
+		telemetry.WithMetricFilter([]string{
+			"http.server." + string(request.IDRequestCount),
+			"http.server." + string(request.IDResponseValidCount),
+			"http.server." + string(request.IDResponseErrorsCount),
+			"http.server." + string(request.IDResponseErrorsTimeout),
+			"http.server." + string(request.IDResponseErrorsRateLimit),
+
+			"grpc.server." + string(request.IDRequestCount),
+			"grpc.server." + string(request.IDResponseValidCount),
+			"grpc.server." + string(request.IDResponseErrorsCount),
+			"grpc.server." + string(request.IDResponseErrorsTimeout),
+			"grpc.server." + string(request.IDResponseErrorsRateLimit),
+		}),
+	)
 	meterProvider := metric.NewMeterProvider(
 		metric.WithReader(exporter),
 		metric.WithReader(metric.NewPeriodicReader(localExporter)),