[add_docker_metadata] log the error instead of returning it (#47760) (#47778)

* go.mod and notice

* go.mod and notice

* prometheus

* fix watcher.Start()

* changelog

* dockerAvailable false

* comments

* unit test

(cherry picked from commit 04732ce)

Co-authored-by: Vihas Makwana <121151420+VihasMakwana@users.noreply.github.com>

Author: mergify[bot]

changelog/fragments/1764007304-bump-elastic-autodiscover.yaml

@@ -0,0 +1,45 @@
+# REQUIRED
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: bug-fix
+
+# REQUIRED for all kinds
+# Change summary; a 80ish characters long description of the change.
+summary: Not being able to start the add_docker_metadata processor is now consistently a non-fatal error when Docker is not available. 
+
+# REQUIRED for breaking-change, deprecation, known-issue
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# description:
+
+# REQUIRED for breaking-change, deprecation, known-issue
+# impact:
+
+# REQUIRED for breaking-change, deprecation, known-issue
+# action:
+
+# REQUIRED for all kinds
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: all
+
+# AUTOMATED
+# OPTIONAL to manually add other PR URLs
+# PR URL: A link the PR that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+# pr: https://github.com/owner/repo/1234
+
+# AUTOMATED
+# OPTIONAL to manually add other issue URLs
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+# issue: https://github.com/owner/repo/1234

libbeat/processors/add_docker_metadata/add_docker_metadata.go

@@ -93,7 +93,9 @@ func buildDockerMetadataProcessor(log *logp.Logger, cfg *conf.C, watcherConstruc
 		dockerAvailable = true
 		log.Debugf("%v: docker environment detected", processorName)
 		if err = watcher.Start(); err != nil {
-			return nil, fmt.Errorf("failed to start watcher: %w", err)
+			// mark dockerAvailable as false because watcher creation failed
+			dockerAvailable = false
+			log.Infof("unable to start the docker watcher: %v", err)
 		}
 	}
 

libbeat/processors/add_docker_metadata/add_docker_metadata_test.go

@@ -20,6 +20,7 @@
 package add_docker_metadata
 
 import (
+	"errors"
 	"fmt"
 	"os"
 	"runtime"
@@ -34,6 +35,7 @@ import (
 	"github.com/elastic/elastic-agent-autodiscover/docker"
 	"github.com/elastic/elastic-agent-libs/config"
 	"github.com/elastic/elastic-agent-libs/logp"
+	"github.com/elastic/elastic-agent-libs/logp/logptest"
 	"github.com/elastic/elastic-agent-libs/mapstr"
 	"github.com/elastic/elastic-agent-system-metrics/metric/system/cgroup"
 	"github.com/elastic/elastic-agent-system-metrics/metric/system/resolve"
@@ -114,7 +116,7 @@ func TestInitializationNoDocker(t *testing.T) {
 func TestInitialization(t *testing.T) {
 	var testConfig = config.NewConfig()
 
-	p, err := buildDockerMetadataProcessor(logp.L(), testConfig, MockWatcherFactory(nil))
+	p, err := buildDockerMetadataProcessor(logp.L(), testConfig, MockWatcherFactory(nil, nil))
 	assert.NoError(t, err, "initializing add_docker_metadata processor")
 
 	input := mapstr.M{}
@@ -130,7 +132,7 @@ func TestNoMatch(t *testing.T) {
 	})
 	assert.NoError(t, err)
 
-	p, err := buildDockerMetadataProcessor(logp.L(), testConfig, MockWatcherFactory(nil))
+	p, err := buildDockerMetadataProcessor(logp.L(), testConfig, MockWatcherFactory(nil, nil))
 	assert.NoError(t, err, "initializing add_docker_metadata processor")
 
 	input := mapstr.M{
@@ -148,7 +150,7 @@ func TestMatchNoContainer(t *testing.T) {
 	})
 	assert.NoError(t, err)
 
-	p, err := buildDockerMetadataProcessor(logp.L(), testConfig, MockWatcherFactory(nil))
+	p, err := buildDockerMetadataProcessor(logp.L(), testConfig, MockWatcherFactory(nil, nil))
 	assert.NoError(t, err, "initializing add_docker_metadata processor")
 
 	input := mapstr.M{
@@ -179,7 +181,7 @@ func TestMatchContainer(t *testing.T) {
 					"b.foo": "3",
 				},
 			},
-		}))
+		}, nil))
 	assert.NoError(t, err, "initializing add_docker_metadata processor")
 
 	input := mapstr.M{
@@ -227,7 +229,7 @@ func TestMatchContainerWithDedot(t *testing.T) {
 					"b.foo": "3",
 				},
 			},
-		}))
+		}, nil))
 	assert.NoError(t, err, "initializing add_docker_metadata processor")
 
 	input := mapstr.M{
@@ -269,7 +271,7 @@ func TestMatchSource(t *testing.T) {
 					"b": "2",
 				},
 			},
-		}))
+		}, nil))
 	assert.NoError(t, err, "initializing add_docker_metadata processor")
 
 	var inputSource string
@@ -328,7 +330,7 @@ func TestDisableSource(t *testing.T) {
 					"b": "2",
 				},
 			},
-		}))
+		}, nil))
 	assert.NoError(t, err, "initializing add_docker_metadata processor")
 
 	input := mapstr.M{
@@ -354,6 +356,7 @@ func TestMatchPIDs(t *testing.T) {
 				},
 			},
 		},
+		nil,
 	))
 	assert.NoError(t, err, "initializing add_docker_metadata processor")
 
@@ -436,22 +439,45 @@ func TestMatchPIDs(t *testing.T) {
 	})
 }
 
+func TestWatcherError(t *testing.T) {
+	logger, observedLogs := logptest.NewTestingLoggerWithObserver(t, "")
+	testConfig, err := config.NewConfigFrom(map[string]interface{}{
+		"match_fields": []string{"foo"},
+	})
+	assert.NoError(t, err)
+
+	p, err := buildDockerMetadataProcessor(logger, testConfig, MockWatcherFactory(nil, errors.New("mock error")))
+	assert.NoError(t, err, "initializing add_docker_metadata processor")
+	assert.Len(t, observedLogs.FilterMessageSnippet("unable to start the docker watcher").TakeAll(), 1)
+
+	input := mapstr.M{
+		"field": "value",
+	}
+	result, err := p.Run(&beat.Event{Fields: input})
+	assert.NoError(t, err, "processing an event")
+	assert.Equal(t, mapstr.M{"field": "value"}, result.Fields)
+}
+
 // Mock container watcher
 
-func MockWatcherFactory(containers map[string]*docker.Container) docker.WatcherConstructor {
+func MockWatcherFactory(containers map[string]*docker.Container, startErr error) docker.WatcherConstructor {
 	if containers == nil {
 		containers = make(map[string]*docker.Container)
 	}
 	return func(_ *logp.Logger, host string, tls *docker.TLSConfig, shortID bool) (docker.Watcher, error) {
-		return &mockWatcher{containers: containers}, nil
+		return &mockWatcher{containers: containers, startErr: startErr}, nil
 	}
 }
 
 type mockWatcher struct {
 	containers map[string]*docker.Container
+	startErr   error
 }
 
 func (m *mockWatcher) Start() error {
+	if m.startErr != nil {
+		return m.startErr
+	}
 	return nil
 }