Allow dynamic datasets and namespaces (elastic#11168)

* allow dynamic datasets and namespaces

* only keep dynamic datasets and namespaces for app metrics and app logs

* add dynamic namespace to every data stream

* add dynamic dataset to traces

* update changelog

apmpackage/apm/changelog.yml

@@ -3,6 +3,12 @@
     - description: Placeholder
       type: enhancement
       link: https://github.com/elastic/apm-server/pull/123
+    - description: Add permissions to reroute to dedicated datasets for logs, metrics and traces
+      type: enhancement
+      link: https://github.com/elastic/apm-server/pull/11168
+    - description: Add permissions to reroute to dedicated namespaces for every data stream.
+      type: enhancement
+      link: https://github.com/elastic/apm-server/pull/11168
 - version: "8.9.0"
   changes:
     - description: Handle `metricset.samples` from apm-server

apmpackage/apm/data_stream/app_logs/manifest.yml

@@ -4,6 +4,9 @@ dataset: apm.app
 dataset_is_prefix: true
 ilm_policy: logs-apm.app_logs-default_policy
 elasticsearch:
+  # Ensures agents have permissions to write data to `logs-*-*`
+  dynamic_dataset: true
+  dynamic_namespace: true
   index_template:
     mappings:
       # TODO(axw) investigate setting `dynamic: runtime`, so that fields are

apmpackage/apm/data_stream/app_metrics/manifest.yml

@@ -4,6 +4,9 @@ dataset: apm.app
 dataset_is_prefix: true
 ilm_policy: metrics-apm.app_metrics-default_policy
 elasticsearch:
+  # Ensures agents have permissions to write data to `metrics-*-*`
+  dynamic_dataset: true
+  dynamic_namespace: true
   index_template:
     mappings:
       # Application metrics must be dynamically mapped,

apmpackage/apm/data_stream/error_logs/manifest.yml

@@ -3,6 +3,7 @@ type: logs
 dataset: apm.error
 ilm_policy: logs-apm.error_logs-default_policy
 elasticsearch:
+  dynamic_namespace: true
   index_template:
     mappings:
       # TODO(axw) investigate setting `dynamic: runtime`, so that fields are

apmpackage/apm/data_stream/internal_metrics/manifest.yml

@@ -3,6 +3,7 @@ type: metrics
 dataset: apm.internal
 ilm_policy: metrics-apm.internal_metrics-default_policy
 elasticsearch:
+  dynamic_namespace: true
   index_template:
     mappings:
       # We are in full control of the field names,

apmpackage/apm/data_stream/rum_traces/manifest.yml

@@ -3,6 +3,7 @@ type: traces
 dataset: apm.rum
 ilm_policy: traces-apm.rum_traces-default_policy
 elasticsearch:
+  dynamic_namespace: true
   index_template:
     mappings:
       # TODO(axw) investigate setting `dynamic: runtime`, so that fields are

apmpackage/apm/data_stream/sampled_traces/manifest.yml

@@ -3,6 +3,7 @@ type: traces
 dataset: apm.sampled
 ilm_policy: traces-apm.sampled_traces-default_policy
 elasticsearch:
+  dynamic_namespace: true
   privileges:
     # We need additional privileges for the sampled traces data stream,
     # for refreshing indices, querying index stats, and reading documents.

apmpackage/apm/data_stream/service_destination_interval_metrics/manifest.yml

@@ -3,6 +3,7 @@ type: metrics
 dataset: apm.service_destination.{{ .Interval }}
 ilm_policy: metrics-apm.service_destination_{{ .Interval }}_metrics-default_policy
 elasticsearch:
+  dynamic_namespace: true
   index_template:
     mappings:
       # We are in full control of the field names,

apmpackage/apm/data_stream/service_summary_interval_metrics/manifest.yml

@@ -3,6 +3,7 @@ type: metrics
 dataset: apm.service_summary.{{ .Interval }}
 ilm_policy: metrics-apm.service_summary_{{ .Interval }}_metrics-default_policy
 elasticsearch:
+  dynamic_namespace: true
   index_template:
     mappings:
       # We are in full control of the field names,

apmpackage/apm/data_stream/service_transaction_interval_metrics/manifest.yml

@@ -3,6 +3,7 @@ type: metrics
 dataset: apm.service_transaction.{{ .Interval }}
 ilm_policy: metrics-apm.service_transaction_{{ .Interval }}_metrics-default_policy
 elasticsearch:
+  dynamic_namespace: true
   index_template:
     mappings:
       # We are in full control of the field names,

apmpackage/apm/data_stream/traces/manifest.yml

@@ -3,6 +3,9 @@ type: traces
 dataset: apm
 ilm_policy: traces-apm.traces-default_policy
 elasticsearch:
+  # Ensures agents have permissions to write data to `traces-*-*`
+  dynamic_dataset: true
+  dynamic_namespace: true
   index_template:
     mappings:
       # TODO(axw) investigate setting `dynamic: runtime`, so that fields are

apmpackage/apm/data_stream/transaction_interval_metrics/manifest.yml

@@ -3,6 +3,7 @@ type: metrics
 dataset: apm.transaction.{{ .Interval }}
 ilm_policy: metrics-apm.transaction_{{ .Interval }}_metrics-default_policy
 elasticsearch:
+  dynamic_namespace: true
   index_template:
     mappings:
       # We are in full control of the field names,

changelogs/head.asciidoc

@@ -20,3 +20,4 @@ https://github.com/elastic/apm-server/compare/8.9\...main[View commits]
 - Add a self-instrumentation transaction to the source map periodic refresh action. {pull}11116[11116]
 - Add a self-instrumentation transaction to the agent config periodic refresh action. {pull}11129[11129]
 - Stop dropping metadata fields from span documents. {pull}11089[11089]
+- Add permissions to reroute events in the integration package. {pull}11168[11168]