[k8s] fix hints stream missing ids (#6485)

* fix: add required ids in hints input streams of type filestream * feat: disable hints default_container_logs when kubernetes integration container logs is enabled (cherry picked from commit 7ad0456)
elastic · Jan 7, 2025 · 0bd0d04 · 0bd0d04
1 parent 8f1e3ec
commit 0bd0d04
Show file tree

Hide file tree

Showing 61 changed files with 82 additions and 3 deletions.
diff --git a/deploy/helm/elastic-agent/examples/kubernetes-hints-autodiscover/rendered/manifest.yaml b/deploy/helm/elastic-agent/examples/kubernetes-hints-autodiscover/rendered/manifest.yaml
@@ -564,6 +564,7 @@ stringData:
     providers:
       kubernetes:
         hints:
+          default_container_logs: false
           enabled: true
         node: ${NODE_NAME}
         scope: node
@@ -1081,7 +1082,7 @@ spec:
       labels:
         name: agent-pernode-example
       annotations:
-        checksum/config: 0df24cb5f7362916ba8cb10621b123918f22f52a7ce9f0b0514c5983de6d06f3
+        checksum/config: daca0d998edb3afa587d96e69b0833f6919ca6ba72f58f3a1f83b22d7e5ffaf6
     spec:
       automountServiceAccountToken: true
       containers:

diff --git a/deploy/helm/elastic-agent/examples/multiple-integrations/rendered/manifest.yaml b/deploy/helm/elastic-agent/examples/multiple-integrations/rendered/manifest.yaml
@@ -590,6 +590,7 @@ stringData:
     providers:
       kubernetes:
         hints:
+          default_container_logs: false
           enabled: true
         node: ${NODE_NAME}
         scope: node
@@ -1107,7 +1108,7 @@ spec:
       labels:
         name: agent-pernode-example
       annotations:
-        checksum/config: 0df24cb5f7362916ba8cb10621b123918f22f52a7ce9f0b0514c5983de6d06f3
+        checksum/config: daca0d998edb3afa587d96e69b0833f6919ca6ba72f58f3a1f83b22d7e5ffaf6
     spec:
       automountServiceAccountToken: true
       containers:

diff --git a/deploy/helm/elastic-agent/templates/integrations/_kubernetes/_preset_pernode.tpl b/deploy/helm/elastic-agent/templates/integrations/_kubernetes/_preset_pernode.tpl
@@ -65,6 +65,11 @@ providers:
   kubernetes:
     hints:
       enabled: true
+{{- if (eq $.Values.kubernetes.containers.logs.enabled false) }}
+      default_container_logs: true
+{{- else }}
+      default_container_logs: false
+{{- end }}
 {{- end -}}
 
 {{- define "elasticagent.kubernetes.pernode.preset.tolerations" -}}

diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/activemq.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/activemq.yml
@@ -5,6 +5,7 @@ inputs:
       use_output: default
       streams:
         - condition: ${kubernetes.hints.activemq.audit.enabled} == true or ${kubernetes.hints.activemq.enabled} == true
+          id: filestream-activemq-audit-${kubernetes.hints.container_id}
           data_stream:
             dataset: activemq.audit
             type: logs
@@ -27,6 +28,7 @@ inputs:
             - forwarded
             - activemq-audit
         - condition: ${kubernetes.hints.activemq.log.enabled} == true or ${kubernetes.hints.activemq.enabled} == true
+          id: filestream-activemq-log-${kubernetes.hints.container_id}
           data_stream:
             dataset: activemq.log
             type: logs

diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/apache.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/apache.yml
@@ -103,6 +103,7 @@ inputs:
       use_output: default
       streams:
         - condition: ${kubernetes.hints.apache.access.enabled} == true or ${kubernetes.hints.apache.enabled} == true
+          id: filestream-apache-access-${kubernetes.hints.container_id}
           data_stream:
             dataset: apache.access
             type: logs
@@ -124,6 +125,7 @@ inputs:
           tags:
             - apache-access
         - condition: ${kubernetes.hints.apache.error.enabled} == true or ${kubernetes.hints.apache.enabled} == true
+          id: filestream-apache-error-${kubernetes.hints.container_id}
           data_stream:
             dataset: apache.error
             type: logs

diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/cassandra.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/cassandra.yml
@@ -5,6 +5,7 @@ inputs:
       use_output: default
       streams:
         - condition: ${kubernetes.hints.cassandra.log.enabled} == true or ${kubernetes.hints.cassandra.enabled} == true
+          id: filestream-cassandra-log-${kubernetes.hints.container_id}
           data_stream:
             dataset: cassandra.log
             type: logs

diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/cef.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/cef.yml
@@ -26,6 +26,7 @@ inputs:
       use_output: default
       streams:
         - condition: ${kubernetes.hints.cef.log.enabled} == true or ${kubernetes.hints.cef.enabled} == true
+          id: filestream-cef-log-${kubernetes.hints.container_id}
           data_stream:
             dataset: cef.log
             type: logs

diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/checkpoint.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/checkpoint.yml
@@ -5,6 +5,7 @@ inputs:
       use_output: default
       streams:
         - condition: ${kubernetes.hints.checkpoint.firewall.enabled} == true or ${kubernetes.hints.checkpoint.enabled} == true
+          id: filestream-checkpoint-firewall-${kubernetes.hints.container_id}
           data_stream:
             dataset: checkpoint.firewall
             type: logs

diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/cockroachdb.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/cockroachdb.yml
@@ -28,6 +28,7 @@ inputs:
       use_output: default
       streams:
         - condition: ${kubernetes.hints.cockroachdb.container_logs.enabled} == true
+          id: filestream-cockroachdb-logs-${kubernetes.hints.container_id}
           data_stream:
             dataset: cockroachdb.container_logs
             type: logs

diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/container_logs.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/container_logs.yml
@@ -5,6 +5,7 @@ inputs:
     use_output: default
     streams:
       - condition: ${kubernetes.hints.container_logs.enabled} == true
+        id: hints-filestream-container-logs-${kubernetes.hints.container_id}
         data_stream:
           dataset: kubernetes.container_logs
           type: logs
@@ -17,4 +18,4 @@ inputs:
         prospector:
           scanner:
             symlinks: true
-    data_stream.namespace: default
+    data_stream.namespace: default
diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/crowdstrike.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/crowdstrike.yml
@@ -5,6 +5,7 @@ inputs:
       use_output: default
       streams:
         - condition: ${kubernetes.hints.crowdstrike.falcon.enabled} == true or ${kubernetes.hints.crowdstrike.enabled} == true
+          id: filestream-crowdstrike-falcon-${kubernetes.hints.container_id}
           data_stream:
             dataset: crowdstrike.falcon
             type: logs
@@ -32,6 +33,7 @@ inputs:
             - forwarded
             - crowdstrike-falcon
         - condition: ${kubernetes.hints.crowdstrike.fdr.enabled} == true or ${kubernetes.hints.crowdstrike.enabled} == true
+          id: filestream-crowdstrike-fdr-${kubernetes.hints.container_id}
           data_stream:
             dataset: crowdstrike.fdr
             type: logs

diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/cyberarkpas.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/cyberarkpas.yml
@@ -39,6 +39,7 @@ inputs:
       use_output: default
       streams:
         - condition: ${kubernetes.hints.cyberarkpas.audit.enabled} == true and ${kubernetes.hints.cyberarkpas.enabled} == true
+          id: filestream-cyberarkpas-audit-${kubernetes.hints.container_id}
           data_stream:
             dataset: cyberarkpas.audit
             type: logs

diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/elasticsearch.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/elasticsearch.yml
@@ -5,6 +5,7 @@ inputs:
       use_output: default
       streams:
         - condition: ${kubernetes.hints.elasticsearch.audit.enabled} == true or ${kubernetes.hints.elasticsearch.enabled} == true
+          id: filestream-elasticsearch-audit-${kubernetes.hints.container_id}
           data_stream:
             dataset: elasticsearch.audit
             type: logs
@@ -49,6 +50,7 @@ inputs:
                     enabled: true
                 symlinks: true
         - condition: ${kubernetes.hints.elasticsearch.deprecation.enabled} == true or ${kubernetes.hints.elasticsearch.enabled} == true
+          id: filestream-elasticsearch-deprecation-${kubernetes.hints.container_id}
           data_stream:
             dataset: elasticsearch.deprecation
             type: logs
@@ -70,6 +72,7 @@ inputs:
                     enabled: true
                 symlinks: true
         - condition: ${kubernetes.hints.elasticsearch.gc.enabled} == true or ${kubernetes.hints.elasticsearch.enabled} == true
+          id: filestream-elasticsearch-gc-${kubernetes.hints.container_id}
           data_stream:
             dataset: elasticsearch.gc
             type: logs
@@ -103,6 +106,7 @@ inputs:
                     enabled: true
                 symlinks: true
         - condition: ${kubernetes.hints.elasticsearch.server.enabled} == true or ${kubernetes.hints.elasticsearch.enabled} == true
+          id: filestream-elasticsearch-server-${kubernetes.hints.container_id}
           data_stream:
             dataset: elasticsearch.server
             type: logs
@@ -125,6 +129,7 @@ inputs:
                     enabled: true
                 symlinks: true
         - condition: ${kubernetes.hints.elasticsearch.slowlog.enabled} == true or ${kubernetes.hints.elasticsearch.enabled} == true
+          id: filestream-elasticsearch-slowlog-${kubernetes.hints.container_id}
           data_stream:
             dataset: elasticsearch.slowlog
             type: logs

diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/endpoint.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/endpoint.yml
@@ -5,6 +5,7 @@ inputs:
       use_output: default
       streams:
         - condition: ${kubernetes.hints.endpoint.container_logs.enabled} == true
+          id: filestream-endpoint-logs-${kubernetes.hints.container_id}
           data_stream:
             dataset: endpoint.container_logs
             type: logs

diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/fireeye.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/fireeye.yml
@@ -5,6 +5,7 @@ inputs:
       use_output: default
       streams:
         - condition: ${kubernetes.hints.fireeye.nx.enabled} == true or ${kubernetes.hints.fireeye.enabled} == true
+          id: filestream-fireeye-nx-${kubernetes.hints.container_id}
           data_stream:
             dataset: fireeye.nx
             type: logs

diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/haproxy.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/haproxy.yml
@@ -5,6 +5,7 @@ inputs:
       use_output: default
       streams:
         - condition: ${kubernetes.hints.haproxy.log.enabled} == true or ${kubernetes.hints.haproxy.enabled} == true
+          id: filestream-haproxy-log-${kubernetes.hints.container_id}
           data_stream:
             dataset: haproxy.log
             type: logs

diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/hashicorp_vault.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/hashicorp_vault.yml
@@ -5,6 +5,7 @@ inputs:
       use_output: default
       streams:
         - condition: ${kubernetes.hints.hashicorp_vault.audit.enabled} == true or ${kubernetes.hints.hashicorp_vault.enabled} == true
+          id: filestream-hashicorp_vault-audit-${kubernetes.hints.container_id}
           data_stream:
             dataset: hashicorp_vault.audit
             type: logs
@@ -26,6 +27,7 @@ inputs:
           tags:
             - hashicorp-vault-audit
         - condition: ${kubernetes.hints.hashicorp_vault.log.enabled} == true or ${kubernetes.hints.hashicorp_vault.enabled} == true
+          id: filestream-hashicorp_vault-log-${kubernetes.hints.container_id}
           data_stream:
             dataset: hashicorp_vault.log
             type: logs

diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/hid_bravura_monitor.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/hid_bravura_monitor.yml
@@ -5,6 +5,7 @@ inputs:
       use_output: default
       streams:
         - condition: ${kubernetes.hints.hid_bravura_monitor.log.enabled} == true or ${kubernetes.hints.hid_bravura_monitor.enabled} == true
+          id: filestream-hid_bravura_monitor-log-${kubernetes.hints.container_id}
           data_stream:
             dataset: hid_bravura_monitor.log
             type: logs

diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/iis.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/iis.yml
@@ -32,6 +32,7 @@ inputs:
       use_output: default
       streams:
         - condition: ${kubernetes.hints.iis.access.enabled} == true or ${kubernetes.hints.iis.enabled} == true
+          id: filestream-iis-access-${kubernetes.hints.container_id}
           data_stream:
             dataset: iis.access
             type: logs
@@ -56,6 +57,7 @@ inputs:
           tags:
             - iis-access
         - condition: ${kubernetes.hints.iis.error.enabled} == true or ${kubernetes.hints.iis.enabled} == true
+          id: filestream-iis-error-${kubernetes.hints.container_id}
           data_stream:
             dataset: iis.error
             type: logs

diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/infoblox_nios.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/infoblox_nios.yml
@@ -5,6 +5,7 @@ inputs:
       use_output: default
       streams:
         - condition: ${kubernetes.hints.infoblox_nios.log.enabled} == true or ${kubernetes.hints.infoblox_nios.enabled} == true
+          id: filestream-infoblox_nios-log-${kubernetes.hints.container_id}
           data_stream:
             dataset: infoblox_nios.log
             type: logs

diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/iptables.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/iptables.yml
@@ -21,6 +21,7 @@ inputs:
       use_output: default
       streams:
         - condition: ${kubernetes.hints.iptables.log.enabled} == true and ${kubernetes.hints.iptables.enabled} == true
+          id: filestream-iptables-log-${kubernetes.hints.container_id}
           data_stream:
             dataset: iptables.log
             type: logs

diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/kafka.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/kafka.yml
@@ -5,6 +5,7 @@ inputs:
       use_output: default
       streams:
         - condition: ${kubernetes.hints.kafka.log.enabled} == true or ${kubernetes.hints.kafka.enabled} == true
+          id: filestream-kafka-log-${kubernetes.hints.container_id}
           data_stream:
             dataset: kafka.log
             type: logs

diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/keycloak.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/keycloak.yml
@@ -5,6 +5,7 @@ inputs:
       use_output: default
       streams:
         - condition: ${kubernetes.hints.keycloak.log.enabled} == true or ${kubernetes.hints.keycloak.enabled} == true
+          id: filestream-keycloak-log-${kubernetes.hints.container_id}
           data_stream:
             dataset: keycloak.log
             type: logs

diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/kibana.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/kibana.yml
@@ -5,6 +5,7 @@ inputs:
       use_output: default
       streams:
         - condition: ${kubernetes.hints.kibana.audit.enabled} == true or ${kubernetes.hints.kibana.enabled} == true
+          id: filestream-kibana-audit-${kubernetes.hints.container_id}
           data_stream:
             dataset: kibana.audit
             type: logs
@@ -24,6 +25,7 @@ inputs:
                     enabled: true
                 symlinks: true
         - condition: ${kubernetes.hints.kibana.log.enabled} == true or ${kubernetes.hints.kibana.enabled} == true
+          id: filestream-kibana-log-${kubernetes.hints.container_id}
           data_stream:
             dataset: kibana.log
             type: logs

diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/log.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/log.yml
@@ -5,6 +5,7 @@ inputs:
       use_output: default
       streams:
         - condition: ${kubernetes.hints.log.container_logs.enabled} == true
+          id: filestream-log-${kubernetes.hints.container_id}
           data_stream:
             dataset: log.container_logs
             type: logs

diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/logstash.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/logstash.yml
@@ -5,6 +5,7 @@ inputs:
       use_output: default
       streams:
         - condition: ${kubernetes.hints.logstash.log.enabled} == true or ${kubernetes.hints.logstash.enabled} == true
+          id: filestream-logstash-log-${kubernetes.hints.container_id}
           data_stream:
             dataset: logstash.log
             type: logs
@@ -34,6 +35,7 @@ inputs:
                     enabled: true
                 symlinks: true
         - condition: ${kubernetes.hints.logstash.slowlog.enabled} == true or ${kubernetes.hints.logstash.enabled} == true
+          id: filestream-logstash-slowlog-${kubernetes.hints.container_id}
           data_stream:
             dataset: logstash.slowlog
             type: logs

diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/mattermost.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/mattermost.yml
@@ -5,6 +5,7 @@ inputs:
       use_output: default
       streams:
         - condition: ${kubernetes.hints.mattermost.audit.enabled} == true or ${kubernetes.hints.mattermost.enabled} == true
+          id: filestream-mattermost-audit-${kubernetes.hints.container_id}
           data_stream:
             dataset: mattermost.audit
             type: logs

diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/microsoft_sqlserver.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/microsoft_sqlserver.yml
@@ -18,6 +18,7 @@ inputs:
       use_output: default
       streams:
         - condition: ${kubernetes.hints.microsoft_sqlserver.log.enabled} == true or ${kubernetes.hints.microsoft_sqlserver.enabled} == true
+          id: filestream-microsoft_sqlserver-log-${kubernetes.hints.container_id}
           data_stream:
             dataset: microsoft_sqlserver.log
             type: logs

diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/mimecast.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/mimecast.yml
@@ -1073,6 +1073,7 @@ inputs:
       use_output: default
       streams:
         - condition: ${kubernetes.hints.mimecast.container_logs.enabled} == true
+          id: filestream-mimecast-logs-${kubernetes.hints.container_id}
           data_stream:
             dataset: mimecast.container_logs
             type: logs

diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/modsecurity.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/modsecurity.yml
@@ -5,6 +5,7 @@ inputs:
       use_output: default
       streams:
         - condition: ${kubernetes.hints.modsecurity.auditlog.enabled} == true or ${kubernetes.hints.modsecurity.enabled} == true
+          id: filestream-modsecurity-auditlog-${kubernetes.hints.container_id}
           data_stream:
             dataset: modsecurity.auditlog
             type: logs

diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/mongodb.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/mongodb.yml
@@ -5,6 +5,7 @@ inputs:
       use_output: default
       streams:
         - condition: ${kubernetes.hints.mongodb.log.enabled} == true or ${kubernetes.hints.mongodb.enabled} == true
+          id: filestream-mongodb-log-${kubernetes.hints.container_id}
           data_stream:
             dataset: mongodb.log
             type: logs

diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/mysql.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/mysql.yml
@@ -5,6 +5,7 @@ inputs:
       use_output: default
       streams:
         - condition: ${kubernetes.hints.mysql.error.enabled} == true or ${kubernetes.hints.mysql.enabled} == true
+          id: filestream-mysql-error-${kubernetes.hints.container_id}
           data_stream:
             dataset: mysql.error
             type: logs
@@ -30,6 +31,7 @@ inputs:
                     enabled: true
                 symlinks: true
         - condition: ${kubernetes.hints.mysql.slowlog.enabled} == true or ${kubernetes.hints.mysql.enabled} == true
+          id: filestream-mysql-slowlog-${kubernetes.hints.container_id}
           data_stream:
             dataset: mysql.slowlog
             type: logs

diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/mysql_enterprise.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/mysql_enterprise.yml
@@ -5,6 +5,7 @@ inputs:
       use_output: default
       streams:
         - condition: ${kubernetes.hints.mysql_enterprise.audit.enabled} == true or ${kubernetes.hints.mysql_enterprise.enabled} == true
+          id: filestream-mysql_enterprise-audit-${kubernetes.hints.container_id}
           data_stream:
             dataset: mysql_enterprise.audit
             type: logs

diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/nats.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/nats.yml
@@ -5,6 +5,7 @@ inputs:
       use_output: default
       streams:
         - condition: ${kubernetes.hints.nats.log.enabled} == true or ${kubernetes.hints.nats.enabled} == true
+          id: filestream-nats-log-${kubernetes.hints.container_id}
           data_stream:
             dataset: nats.log
             type: logs