Merge branch 'main' into dans_final_factor

.buildkite/x-pack/pipeline.xpack.heartbeat.yml

@@ -111,7 +111,7 @@ steps:
 
       - label: ":windows: x-pack/heartbeat: Win 2016 Unit Tests"
         key: "mandatory-win-2016-unit-tests"
-        skip: "skipping due to elastic/beats#23957 and elastic/beats#23958"
+        skip: "skipping due missing deps, elastic/ingest-dev#3844"
         command: |
           Set-Location -Path x-pack/heartbeat
           mage build test
@@ -134,7 +134,6 @@ steps:
       # Doesn't exist in Jenkins
       - label: ":windows: x-pack/heartbeat: Win 2022 Unit Tests"
         key: "mandatory-win-2022-unit-tests"
-        skip: "skipping due to elastic/beats#23957 and elastic/beats#23958"
         command: |
           Set-Location -Path x-pack/heartbeat
           mage build unitTest
@@ -157,7 +156,6 @@ steps:
   - group: "Extended Windows Tests"
     key: "x-pack-heartbeat-extended-win-tests"
     if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/
-    skip: "skipping due to elastic/beats#23957 and elastic/beats#23958"
 
     steps:
       - label: ":windows: x-pack/heartbeat: Win 10 Unit Tests"

.mergify.yml

@@ -341,3 +341,16 @@ pull_request_rules:
         labels:
           - "backport"
         title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"
+  - name: backport patches to 8.x branch
+    conditions:
+      - merged
+      - label=backport-v8.x
+    actions:
+      backport:
+        assignees:
+          - "{{ author }}"
+        branches:
+          - "8.x"
+        labels:
+          - "backport"
+        title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"

CHANGELOG-developer.next.asciidoc

@@ -202,6 +202,7 @@ The list below covers the major changes between 7.0.0-rc2 and main only.
 - Bump version of elastic/toutoumomoma to remove internal forks of stdlib debug packages. {pull}40325[40325]
 - Refactor x-pack/filebeat/input/websocket for generalisation. {pull}40308[40308]
 - Add a configuration option for TCP/UDP network type. {issue}40407[40407] {pull}40623[40623]
+- Added debug logging to parquet reader in x-pack/libbeat/reader. {pull}40651[40651]
 
 ==== Deprecated
 

CHANGELOG.next.asciidoc

@@ -164,6 +164,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Fix the "No such input type exist: 'azure-eventhub'" error on the Windows platform {issue}40608[40608] {pull}40609[40609]
 - awss3 input: Fix handling of SQS notifications that don't contain a region. {pull}40628[40628]
 - Fix credential handling when workload identity is being used in GCS input. {issue}39977[39977] {pull}40663[40663]
+- Fix publication of group data from the Okta entity analytics provider. {pull}40681[40681]
 
 *Heartbeat*
 
@@ -293,8 +294,10 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Improve logging in Okta Entity Analytics provider. {issue}40106[40106] {pull}40347[40347]
 - Document `winlog` input. {issue}40074[40074] {pull}40462[40462]
 - Added retry logic to websocket connections in the streaming input. {issue}40271[40271] {pull}40601[40601]
+- Add new metricset cluster for the vSphere module. {pull}40536[40536]
 - Disable event normalization for netflow input {pull}40635[40635]
 - Allow attribute selection in the Active Directory entity analytics provider. {issue}40482[40482] {pull}40662[40662]
+- Improve error quality when CEL program does not correctly return an events array. {pull}40580[40580]
 
 *Auditbeat*
 
@@ -323,6 +326,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Add SSL support for aerospike module {pull}38126[38126]
 - Add new metricset network for the vSphere module. {pull}40559[40559]
 - Add new metricset resourcepool for the vSphere module. {pull}40456[40456]
+- Log the total time taken for GCP `ListTimeSeries` and `AggregatedList` requests {pull}40661[40661]
 
 *Metricbeat*
 

dev-tools/packaging/templates/docker/Dockerfile.tmpl

@@ -109,7 +109,36 @@ RUN readlink -f {{ $beatBinary }} | xargs setcap {{ .linux_capabilities }}
 RUN groupadd --gid 1000 {{ .BeatName }}
 RUN useradd -M --uid 1000 --gid 1000 --groups 0 --home {{ $beatHome }} {{ .user }}
 
-{{- if (and (eq .BeatName "heartbeat") (not (contains .from "ubi-minimal"))) }}
+
+{{- if (and (eq .BeatName "heartbeat") (contains .from "wolfi"))  }}
+# Install required dependencies from wolfi repository
+RUN apk add --no-interactive --no-progress nodejs-18=18.20.4-r0 npm=10.8.3-r0 glib dbus-glib libatk-1.0 \
+    libatk-bridge-2.0 cups-libs libxcomposite libxdamage libxrandr libxkbcommon pango alsa-lib \
+    font-opensans fontconfig gtk icu-data-full libnss mesa font-noto-cjk font-noto-emoji
+
+USER {{ .user }}
+
+# Setup synthetics env vars
+ENV ELASTIC_SYNTHETICS_CAPABLE=true
+ENV TZ=UTC
+ENV NPM_CONFIG_PREFIX={{ $beatHome }}/.npm
+ENV PATH="$NPM_CONFIG_PREFIX/bin:$PATH"
+
+RUN echo \
+    $NPM_CONFIG_PREFIX \
+    {{ $beatHome }}/.config \
+    {{ $beatHome }}/.synthetics \
+    {{ $beatHome }}/.npm \
+    {{ $beatHome }}/.cache \
+    | xargs -IDIR sh -c 'mkdir -m 0775 -p DIR'
+
+RUN (npm i -g --loglevel verbose --production --engine-strict  @elastic/synthetics@stack_release || sh -c 'tail -n +1 ${NPM_CONFIG_PREFIX}/_logs/* && exit 1') 
+
+USER root
+
+{{- end }}
+
+{{- if (and (eq .BeatName "heartbeat") (contains .from "ubuntu")) }}
 ENV NODE_PATH={{ $beatHome }}/.node
 RUN echo \
     $NODE_PATH \
@@ -122,8 +151,7 @@ RUN echo \
 # Setup synthetics env vars
 ENV ELASTIC_SYNTHETICS_CAPABLE=true
 ENV TZ=UTC
-ENV SUITES_DIR={{ $beatHome }}/suites
-ENV NODE_VERSION=18.20.3
+ENV NODE_VERSION=18.20.4
 ENV PATH="$NODE_PATH/node/bin:$PATH"
 # Install the latest version of @elastic/synthetics forcefully ignoring the previously
 # cached node_modules, heartbeat then calls the global executable to run test suites

filebeat/input/filestream/internal/input-logfile/harvester_test.go

@@ -31,9 +31,9 @@ import (
 
 	"github.com/elastic/beats/v7/filebeat/input/filestream/internal/task"
 	input "github.com/elastic/beats/v7/filebeat/input/v2"
+	"github.com/elastic/beats/v7/libbeat/beat"
 	"github.com/elastic/beats/v7/libbeat/common/atomic"
 	"github.com/elastic/beats/v7/libbeat/tests/resources"
-	"github.com/elastic/beats/v7/x-pack/dockerlogbeat/pipelinemock"
 	"github.com/elastic/elastic-agent-libs/logp"
 )
 
@@ -393,7 +393,7 @@ func TestDefaultHarvesterGroup(t *testing.T) {
 func testDefaultHarvesterGroup(t *testing.T, mockHarvester Harvester) *defaultHarvesterGroup {
 	return &defaultHarvesterGroup{
 		readers:    newReaderGroup(),
-		pipeline:   &pipelinemock.MockPipelineConnector{},
+		pipeline:   &MockPipeline{},
 		harvester:  mockHarvester,
 		store:      testOpenStore(t, "test", nil),
 		identifier: &sourceIdentifier{"filestream::.global::"},
@@ -465,3 +465,71 @@ func (tl *testLogger) Errorf(format string, args ...interface{}) {
 func (tl *testLogger) String() string {
 	return (*strings.Builder)(tl).String()
 }
+
+// MockClient is a mock implementation of the beat.Client interface.
+type MockClient struct {
+	published []beat.Event // Slice to store published events
+
+	closed bool       // Flag to indicate if the client is closed
+	mu     sync.Mutex // Mutex to synchronize access to the published events slice
+}
+
+// GetEvents returns all the events published by the mock client.
+func (m *MockClient) GetEvents() []beat.Event {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	return m.published
+}
+
+// Publish publishes a single event.
+func (m *MockClient) Publish(e beat.Event) {
+	es := make([]beat.Event, 1)
+	es = append(es, e)
+
+	m.PublishAll(es)
+}
+
+// PublishAll publishes multiple events.
+func (m *MockClient) PublishAll(es []beat.Event) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	m.published = append(m.published, es...)
+}
+
+// Close closes the mock client.
+func (m *MockClient) Close() error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	if m.closed {
+		return fmt.Errorf("mock already closed")
+	}
+
+	m.closed = true
+	return nil
+}
+
+// MockPipeline is a mock implementation of the beat.Pipeline interface.
+type MockPipeline struct {
+	c  beat.Client // Client used by the pipeline
+	mu sync.Mutex  // Mutex to synchronize access to the client
+}
+
+// ConnectWith connects the mock pipeline with a client using the provided configuration.
+func (mp *MockPipeline) ConnectWith(config beat.ClientConfig) (beat.Client, error) {
+	mp.mu.Lock()
+	defer mp.mu.Unlock()
+
+	c := &MockClient{}
+
+	mp.c = c
+
+	return c, nil
+}
+
+// Connect connects the mock pipeline with a client using the default configuration.
+func (mp *MockPipeline) Connect() (beat.Client, error) {
+	return mp.ConnectWith(beat.ClientConfig{})
+}

filebeat/input/journald/pkg/journalfield/conv_expand_test.go

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-//go:build linux && cgo
+//go:build linux
 
 package journalfield
 

filebeat/input/journald/pkg/journalfield/conv_test.go

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-//go:build linux && cgo
+//go:build linux
 
 package journalfield
 

filebeat/input/journald/pkg/journalfield/default.go

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-//go:build linux && cgo
+//go:build linux
 
 package journalfield
 

filebeat/input/journald/pkg/journalfield/matcher_test.go

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-//go:build linux && cgo
+//go:build linux
 
 package journalfield
 

libbeat/cfgfile/cfgfile.go

@@ -18,6 +18,7 @@
 package cfgfile
 
 import (
+	"flag"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -97,6 +98,8 @@ func GetDefaultCfgfile() string {
 }
 
 // HandleFlags adapts default config settings based on command line flags.
+// This also stores if -E management.enabled=true was set on command line
+// to determine if running the Beat under agent.
 func HandleFlags() error {
 	// default for the home path is the binary location
 	home, err := filepath.Abs(filepath.Dir(os.Args[0]))
@@ -114,6 +117,27 @@ func HandleFlags() error {
 		common.PrintConfigDebugf(overwrites, "CLI setting overwrites (-E flag):")
 	}
 
+	// Enable check to see if beat is running under Agent
+	// This is stored in a package so the modules which don't have
+	// access to the config can check this value.
+	type management struct {
+		Enabled bool `config:"management.enabled"`
+	}
+	var managementSettings management
+	cfgFlag := flag.Lookup("E")
+	if cfgFlag == nil {
+		fleetmode.SetAgentMode(false)
+		return nil
+	}
+	cfgObject, _ := cfgFlag.Value.(*config.SettingsFlag)
+	cliCfg := cfgObject.Config()
+
+	err = cliCfg.Unpack(&managementSettings)
+	if err != nil {
+		fleetmode.SetAgentMode(false)
+		return nil //nolint:nilerr // unpacking failing isn't an error for this case
+	}
+	fleetmode.SetAgentMode(managementSettings.Enabled)
 	return nil
 }
 

libbeat/common/fleetmode/fleet_mode.go

@@ -17,33 +17,18 @@
 
 package fleetmode
 
-import (
-	"flag"
-
-	"github.com/elastic/elastic-agent-libs/config"
-)
+var managementEnabled bool
+
+// SetAgentMode stores if the Beat is running under Elastic Agent.
+// Normally this is called when the command line flags are parsed.
+// This is stored as a package level variable because some components
+// (like filebeat/metricbeat modules) don't have access to the
+// configuration information to determine this on their own.
+func SetAgentMode(enabled bool) {
+	managementEnabled = enabled
+}
 
-// Enabled checks to see if filebeat/metricbeat is running under Agent
-// The management setting is stored in the main Beat runtime object, but we can't see that from a module
-// So instead we check the CLI flags, since Agent starts filebeat/metricbeat with "-E", "management.enabled=true"
+// Enabled returns true if the Beat is running under Elastic Agent.
 func Enabled() bool {
-	type management struct {
-		Enabled bool `config:"management.enabled"`
-	}
-	var managementSettings management
-
-	cfgFlag := flag.Lookup("E")
-	if cfgFlag == nil {
-		return false
-	}
-
-	cfgObject, _ := cfgFlag.Value.(*config.SettingsFlag)
-	cliCfg := cfgObject.Config()
-
-	err := cliCfg.Unpack(&managementSettings)
-	if err != nil {
-		return false
-	}
-
-	return managementSettings.Enabled
+	return managementEnabled
 }