Merge branch 'main' of github.com:elastic/beats into 40185-update-inc…

…lude-matches

.buildkite/packaging.pipeline.yml

@@ -61,7 +61,7 @@ steps:
 
       - label: Staging dashboards
         # TODO remove OR clause below (see earlier comment)
-        if: build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env('VERSION_QUALIFIER') != null
+        if: (build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env('VERSION_QUALIFIER') != null) || build.env('RUN_STAGING') == "true"
         depends_on: start-gate-staging
         key: dashboards-staging
         # TODO: container with go and make
@@ -73,9 +73,10 @@ steps:
         retry:
           automatic:
             - limit: 1
-        commands:
-          - make build/distributions/dependencies.csv
-          - make beats-dashboards
+        commands: |
+          source .buildkite/scripts/version_qualifier.sh
+          make build/distributions/dependencies.csv
+          make beats-dashboards
         env:
           SNAPSHOT: false
           DEV: false
@@ -174,14 +175,16 @@ steps:
     key: packaging-staging
     depends_on: start-gate-staging
     # TODO remove OR clause below (see earlier comment)
-    if: build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env('VERSION_QUALIFIER') != null
+    if: (build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env('VERSION_QUALIFIER') != null) || build.env('RUN_STAGING') == "true"
     steps:
       - label: "STAGING: {{matrix}}"
         env:
           PLATFORMS: "${PLATFORMS}"
           SNAPSHOT: false
           DEV: false
-        command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}"
+        command: |
+          source .buildkite/scripts/version_qualifier.sh
+          .buildkite/scripts/packaging/package-dra.sh {{matrix}}
         agents:
           provider: gcp
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -214,7 +217,9 @@ steps:
           PACKAGES: "docker"
           SNAPSHOT: false
           DEV: false
-        command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}"
+        command: |
+          source .buildkite/scripts/version_qualifier.sh
+          .buildkite/scripts/packaging/package-dra.sh {{matrix}}
         agents:
           provider: "aws"
           imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
@@ -244,7 +249,9 @@ steps:
           PLATFORMS: "${PLATFORMS}"
           SNAPSHOT: false
           DEV: false
-        command: ".buildkite/scripts/packaging/package-dra.sh x-pack/agentbeat"
+        command: |
+          source .buildkite/scripts/version_qualifier.sh
+          .buildkite/scripts/packaging/package-dra.sh x-pack/agentbeat
         agents:
           provider: gcp
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -280,7 +287,7 @@ steps:
 
       - label: DRA Staging
         # TODO remove OR clause below (see earlier comment)
-        if: build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env('VERSION_QUALIFIER') != null
+        if: (build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env('VERSION_QUALIFIER') != null) || build.env('RUN_STAGING') == "true"
         key: dra-staging
         env:
           DRA_WORKFLOW: staging
@@ -289,6 +296,7 @@ steps:
           - packaging-staging
           - dashboards-staging
         command: |
+          source .buildkite/scripts/version_qualifier.sh
           buildkite-agent artifact download "build/**" .
           .buildkite/scripts/packaging/prepare-release-manager.sh staging
           .buildkite/scripts/dra.sh

.buildkite/scripts/version_qualifier.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+# An opinionated approach to managing the Elastic Qualifier for the DRA in a Google Bucket
+# instead of using a Buildkite env variable.
+
+if [[ -n "$VERSION_QUALIFIER" ]]; then
+  echo "~~~ VERSION_QUALIFIER externally set to [$VERSION_QUALIFIER]"
+  return 0
+fi
+
+# DRA_BRANCH can be used for manually testing packaging with PRs
+# e.g. define `DRA_BRANCH="main"` under Options/Environment Variables in the Buildkite UI after clicking new Build
+BRANCH="${DRA_BRANCH:="${BUILDKITE_BRANCH:=""}"}"
+
+qualifier=""
+URL="https://storage.googleapis.com/dra-qualifier/${BRANCH}"
+if curl -sf -o /dev/null "$URL" ; then
+  qualifier=$(curl -s "$URL")
+fi
+
+export VERSION_QUALIFIER="$qualifier"
+echo "~~~ VERSION_QUALIFIER set to [$VERSION_QUALIFIER]"

.mergify.yml

@@ -147,20 +147,6 @@ pull_request_rules:
           branches, such as:
           * `backport-8./d` is the label to automatically backport to the `8./d` branch. `/d` is the digit
 
-  - name: add backport-8.x label for main only if no skipped or assigned already
-    conditions:
-      - -label~=^(backport-skip|backport-8.x)$
-      - base=main
-      - -merged
-      - -closed
-    actions:
-      comment:
-        message: |
-          `backport-8.x` has been added to help with the transition to the new branch `8.x`.
-          If you don't need it please use `backport-skip` label and remove the `backport-8.x` label.
-      label:
-        add:
-          - backport-8.x
   - name: notify the backport has not been merged yet
     conditions:
       - -merged

CHANGELOG.next.asciidoc

@@ -59,6 +59,13 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Filestream inputs can define `allow_deprecated_id_duplication: true` to run keep the previous behaviour of running inputs with duplicated IDs. {issue}41938[41938] {pull}41954[41954]
 - The Filestream input only starts to ingest a file when it is >= 1024 bytes in size. This happens because the fingerprint` is the default file identity now. To restore the previous behaviour, set `file_identity.native: ~` and `prospector.scanner.fingerprint.enabled: false` {issue}40197[40197] {pull}41762[41762]
 - Filebeat fails to start when its configuration contains usage of the deprecated `log` or `container` inputs. However, they can still be using while setting `allow_deprecated_use: true` in their configuration {pull}42295[42295]
+- The fields produced by the Journald input are updated to better match ECS. Renamed fields:
+Dropped fields: `syslog.priority` and `syslog.facility` while keeping their duplicated equivalent:
+`log.syslog.priority`,`log.syslog.facility.code`. Renamed fields: `syslog.identifier` -> `log.syslog.appname`,
+`syslog.pid` -> `log.syslog.procid`. `container.id_truncated` is dropped because the full container ID is
+already present as `container.id` and `container.log.tag` is dropped because it is already present as
+`log.syslog.appname`. The field `container.partial` is replaced by the tag `partial_message` if it was `true`,
+otherwise no tag is added. {issue}42208[42208] {pull}42403[42403]
 
 *Heartbeat*
 
@@ -402,6 +409,8 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Added OAuth2 support with auto token refresh for websocket streaming input. {issue}41989[41989] {pull}42212[42212]
 - Added infinite & blanket retry options to websockets and improved logging and retry logic. {pull}42225[42225]
 - Introduce ignore older and start timestamp filters for AWS S3 input. {pull}41804[41804]
+- Journald input now can report its status to Elastic-Agent {issue}39791[39791] {pull}42462[42462]
+- Publish events progressively in the Okta provider of the Entity Analytics input. {issue}40106[40106] {pull}42567[42567]
 
 *Auditbeat*
 
@@ -461,6 +470,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Preserve queries for debugging when `merge_results: true` in SQL module {pull}42271[42271]
 - Collect more fields from ES node/stats metrics and only those that are necessary {pull}42421[42421]
 - Add new metricset wmi for the windows module. {pull}42017[42017]
+- Update beat module with apm-server tail sampling monitoring metrics fields {pull}42569[42569]
 
 *Metricbeat*
 - Add benchmark module {pull}41801[41801]