Merge branch 'main' into add_gateway_collector

.github/workflows/bump-golang.yml

@@ -22,7 +22,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Install Updatecli in the runner
-        uses: updatecli/updatecli-action@1e7fcb1c5df159bca56c24c5335a050d611fbfdd # 2.74.0
+        uses: updatecli/updatecli-action@4aca518a70708e38063453d8de9c551af7f48ac3 # 2.75.0
 
       - name: Run Updatecli in Apply mode
         run: updatecli apply --config .ci/updatecli/updatecli-bump-golang.yml --values .ci/updatecli/values.d/scm.yml

_meta/config/common.p2.yml.tmpl

@@ -52,6 +52,16 @@ inputs:
 #         paths:
 #           - /var/log/*.log
 
+#   # Collecting journald logs, including coredumps.
+#   - type: journald
+#     # Input ID allowing Elastic Agent to track the state of this input. Must be unique.
+#     id: your-journald-input-id
+#     streams:
+#       # Stream ID for this data stream allowing Filebeat to track the state of the ingested data. Must be unique.
+#       # Each journald data stream creates a separate instance of the Filebeat journald input.
+#       - id: your-journald-stream-id
+#         data_stream:
+#           dataset: generic
 
 # agent.monitoring:
 #   # enabled turns on monitoring of running processes

_meta/config/common.reference.p2.yml.tmpl

@@ -50,6 +50,17 @@ inputs:
 #         paths:
 #           - /var/log/*.log
 
+#   # Collecting journald logs, including coredumps.
+#   - type: journald
+#     # Input ID allowing Elastic Agent to track the state of this input. Must be unique.
+#     id: your-journald-input-id
+#     streams:
+#       # Stream ID for this data stream allowing Filebeat to track the state of the ingested data. Must be unique.
+#       # Each journald data stream creates a separate instance of the Filebeat journald input.
+#       - id: your-journald-stream-id
+#         data_stream:
+#           dataset: generic
+
 # management:
 #   # Mode of management, the Elastic Agent support two modes of operation:
 #   #

changelog/fragments/1733248787-flag-to-skip-fleet-audit.yaml

@@ -0,0 +1,32 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: enhancement
+
+# Change summary; a 80ish characters long description of the change.
+summary: Add a flag to skip audit/unenroll call to fleet server during uninstall
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+description: This change adds a flag to skip audit/unenroll call to fleet server. While uninstalling elastic-agent it tries to notify fleet server about the uninstallation. But in somecases users might know that the fleet server is unreachable and this notification logs multiple failures continuously. Adding this flag skips this call.
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: "elastic-agent"
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+#pr: https://github.com/owner/repo/1234
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+#issue: https://github.com/owner/repo/1234

changelog/fragments/1733862556-Log-warning-on-same-version-upgrade-attempts.yaml

@@ -0,0 +1,34 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: bug-fix
+
+# Change summary; a 80ish characters long description of the change.
+summary: Log warning on same version upgrade attempts
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+description: |
+  Log a warning instead of reporting an error whan a same-version upgrade is
+  attempted. This prevents the agent from reporting a "failed" status.
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: elastic-agent
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+#pr: https://github.com/owner/repo/1234
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+issue: https://github.com/elastic/elastic-agent/issues/6186

changelog/fragments/1734023789-add-loadbalancing-exporter-to-EDOT-collector.yaml

@@ -0,0 +1,32 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: feature
+
+# Change summary; a 80ish characters long description of the change.
+summary: add loadbalancing exporter to EDOT collector
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+#description:
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: "elastic-agent"
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+#pr: https://github.com/owner/repo/1234
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+#issue: https://github.com/owner/repo/1234

changelog/fragments/1734626964-Windows-add-Event-Logging-to-install,-uninstall,-enroll.yaml

@@ -0,0 +1,32 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: enhancement
+
+# Change summary; a 80ish characters long description of the change.
+summary: Windows add Event Logging to install, uninstall, enroll failures
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+#description:
+
+# Affected component; a word indicating the component this changeset affects.
+component: elastic-agent
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: https://github.com/elastic/elastic-agent/6410
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+issue: https://github.com/elastic/elastic-agent/6338

changelog/fragments/1735137195-remove-deprecated-list-in-favor-of-items.yaml

@@ -0,0 +1,31 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: feature
+
+# Change summary; a 80ish characters long description of the change.
+summary: removes `list` from kibanaFetchToken in favor of `items` as the former is deprecated and will be removed from the api response
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+#description:
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: "elastic-agent"
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: https://github.com/elastic/elastic-agent/pull/6437
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+issue: https://github.com/elastic/elastic-agent/issues/6023

changelog/fragments/1735664420-remove-deprecated-path-install-cli-flag.yaml

@@ -0,0 +1,30 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: breaking-change
+
+# Change summary; a 80ish characters long description of the change.
+summary: Removing --path.install option
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+#description:
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: "elastic-agent"
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: https://github.com/elastic/elastic-agent/pull/6461/files
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+issue: https://github.com/elastic/elastic-agent/issues/2489

deploy/helm/elastic-agent/examples/netflow-service/README.md

@@ -0,0 +1,28 @@
+# Example: Netflow Custom Integration
+
+In this example we define a `netflow` custom integration alongside a custom agent preset defined in [agent-netflow-values.yaml](agent-netflow-values.yaml). Also, we disable all `kubernetes` related providers and creation of cluster role and service account, as they are not required for this example.
+
+## Prerequisites:
+1. A k8s secret that contains the connection details to an Elasticsearch cluster such as the URL and the API key ([Kibana - Creating API Keys](https://www.elastic.co/guide/en/kibana/current/api-keys.html)):
+    ```console
+    kubectl create secret generic es-api-secret \
+       --from-literal=api_key=... \
+       --from-literal=url=...
+    ```
+
+2. `NetFlow Records` integration assets are installed through Kibana
+
+## Run:
+1. Install Helm chart
+    ```console
+    helm install elastic-agent ../../ -f ./agent-netflow-values.yaml
+    ```
+
+2. Run the netflow data generator deployment
+    ```console
+   kubectl run -it --rm netflow-generator --image=networkstatic/nflow-generator --restart=Never -- -t agent-netflow-elastic-agent.default.svc.cluster.local -p 2055
+    ```
+
+## Validate:
+
+1. The Kibana `netflow`-related dashboards should start showing netflow related data.

deploy/helm/elastic-agent/examples/netflow-service/agent-netflow-values.yaml

@@ -0,0 +1,62 @@
+outputs:
+  default:
+    type: ESSecretAuthAPI
+    secretName: es-api-secret
+
+extraIntegrations:
+  netflow:
+    id: netflow-netflow-60a9d5b2-c611-4749-90bf-5e2443936c1d
+    name: netflow-1
+    preset: netflow
+    revision: 1
+    type: netflow
+    use_output: default
+    meta:
+      package:
+        name: netflow
+        version: 2.19.1
+    data_stream:
+      namespace: default
+    package_policy_id: 60a9d5b2-c611-4749-90bf-5e2443936c1d
+    streams:
+      - id: netflow-netflow.log-60a9d5b2-c611-4749-90bf-5e2443936c1d
+        data_stream:
+          dataset: netflow.log
+          type: logs
+        protocols:
+          - v1
+          - v5
+          - v6
+          - v7
+          - v8
+          - v9
+          - ipfix
+        host: '0.0.0.0:2055'
+        max_message_size: 10KiB
+        expiration_timeout: 30m
+        queue_size: 8192
+        detect_sequence_reset: true
+        tags:
+          - netflow
+          - forwarded
+        publisher_pipeline.disable_host: true
+
+kubernetes:
+  enabled: false
+
+agent:
+  unprivileged: true
+  presets:
+    netflow:
+      automountServiceAccountToken: false
+      mode: deployment
+      service:
+        type: ClusterIP
+      ports:
+        - containerPort: 2055
+          servicePort: 2055
+          protocol: UDP
+      serviceAccount:
+        create: false
+      clusterRole:
+        create: false