Update documentation and notice

Update documentation and notice file

NOTICE.txt

@@ -12700,12 +12700,12 @@ SOFTWARE
 
 
 --------------------------------------------------------------------------------
-Dependency : github.com/elastic/elastic-agent-libs
-Version: v0.7.3
+Dependency : github.com/belimawr/elastic-agent-libs
+Version: v0.2.9-0.20240116105334-25f61a14ad41
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.7.3/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/belimawr/elastic-agent-libs@v0.2.9-0.20240116105334-25f61a14ad41/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004

auditbeat/auditbeat.reference.yml

@@ -1544,6 +1544,42 @@ logging.files:
   # file. Defaults to true.
   # rotateonstartup: true
 
+# Some outputs will log raw events on errors like indexing errors in the
+# Elasticsearch output, to prevent logging raw events together with other
+# log messages, a different log file, only for log entries containing raw events,
+# is used. It will use the same level, selectors and all other configurations
+# from the default logger, but it will have it's own file configuration.
+#logging.events:
+  #files:
+    # Configure the path where the logs are written. The default is the logs directory
+    # under the home path (the binary location).
+    #path: /var/log/auditbeat
+
+    # The name of the files where the logs are written to.
+    #name: auditbeat-events-data
+
+    # Configure log file size limit. If the limit is reached, log file will be
+    # automatically rotated.
+    #rotateeverybytes: 10485760 # = 10MB
+
+    # Number of rotated log files to keep. The oldest files will be deleted first.
+    #keepfiles: 7
+
+    # The permissions mask to apply when rotating log files. The default value is 0600.
+    # Must be a valid Unix-style file permissions mask expressed in octal notation.
+    #permissions: 0600
+
+    # Enable log file rotation on time intervals in addition to the size-based rotation.
+    # Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h
+    # are boundary-aligned with minutes, hours, days, weeks, months, and years as
+    # reported by the local system clock. All other intervals are calculated from the
+    # Unix epoch. Defaults to disabled.
+    #interval: 0
+
+    # Rotate existing logs on startup rather than appending them to the existing
+    # file. Defaults to true.
+    # rotateonstartup: true
+
 # ============================= X-Pack Monitoring ==============================
 # Auditbeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

auditbeat/auditbeat.yml

@@ -169,6 +169,20 @@ processors:
 # "publisher", "service".
 #logging.selectors: ["*"]
 
+# Some outputs will log raw events on errors like indexing errors in the
+# Elasticsearch output, to prevent logging raw events together with other
+# log messages, a different log file, only for log entries containing raw events,
+# is used. It will use the same level, selectors and all other configurations
+# from the default logger, but it will have it's own file configuration.
+#logging.events:
+  #files:
+    # Configure the path where the logs are written. The default is the logs directory
+    # under the home path (the binary location).
+    #path: /var/log/auditbeat
+
+    # The name of the files where the logs are written to.
+    #name: auditbeat-events-data
+
 # ============================= X-Pack Monitoring ==============================
 # Auditbeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

filebeat/filebeat.reference.yml

@@ -2640,6 +2640,42 @@ logging.files:
   # file. Defaults to true.
   # rotateonstartup: true
 
+# Some outputs will log raw events on errors like indexing errors in the
+# Elasticsearch output, to prevent logging raw events together with other
+# log messages, a different log file, only for log entries containing raw events,
+# is used. It will use the same level, selectors and all other configurations
+# from the default logger, but it will have it's own file configuration.
+#logging.events:
+  #files:
+    # Configure the path where the logs are written. The default is the logs directory
+    # under the home path (the binary location).
+    #path: /var/log/filebeat
+
+    # The name of the files where the logs are written to.
+    #name: filebeat-events-data
+
+    # Configure log file size limit. If the limit is reached, log file will be
+    # automatically rotated.
+    #rotateeverybytes: 10485760 # = 10MB
+
+    # Number of rotated log files to keep. The oldest files will be deleted first.
+    #keepfiles: 7
+
+    # The permissions mask to apply when rotating log files. The default value is 0600.
+    # Must be a valid Unix-style file permissions mask expressed in octal notation.
+    #permissions: 0600
+
+    # Enable log file rotation on time intervals in addition to the size-based rotation.
+    # Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h
+    # are boundary-aligned with minutes, hours, days, weeks, months, and years as
+    # reported by the local system clock. All other intervals are calculated from the
+    # Unix epoch. Defaults to disabled.
+    #interval: 0
+
+    # Rotate existing logs on startup rather than appending them to the existing
+    # file. Defaults to true.
+    # rotateonstartup: true
+
 # ============================= X-Pack Monitoring ==============================
 # Filebeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

filebeat/filebeat.yml

@@ -186,6 +186,20 @@ processors:
 # "publisher", "service".
 #logging.selectors: ["*"]
 
+# Some outputs will log raw events on errors like indexing errors in the
+# Elasticsearch output, to prevent logging raw events together with other
+# log messages, a different log file, only for log entries containing raw events,
+# is used. It will use the same level, selectors and all other configurations
+# from the default logger, but it will have it's own file configuration.
+#logging.events:
+  #files:
+    # Configure the path where the logs are written. The default is the logs directory
+    # under the home path (the binary location).
+    #path: /var/log/filebeat
+
+    # The name of the files where the logs are written to.
+    #name: filebeat-events-data
+
 # ============================= X-Pack Monitoring ==============================
 # Filebeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

go.mod

@@ -420,4 +420,4 @@ replace (
 // Exclude this version because the version has an invalid checksum.
 exclude github.com/docker/distribution v2.8.0+incompatible
 
-replace github.com/elastic/elastic-agent-libs => github.com/belimawr/elastic-agent-libs v0.2.9-0.20231221105324-aedb70a4f832
+replace github.com/elastic/elastic-agent-libs => github.com/belimawr/elastic-agent-libs v0.2.9-0.20240116105334-25f61a14ad41

go.sum

@@ -373,8 +373,8 @@ github.com/awslabs/goformation/v4 v4.1.0 h1:JRxIW0IjhYpYDrIZOTJGMu2azXKI+OK5dP56
 github.com/awslabs/goformation/v4 v4.1.0/go.mod h1:MBDN7u1lMNDoehbFuO4uPvgwPeolTMA2TzX1yO6KlxI=
 github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5 h1:lxW5Q6K2IisyF5tlr6Ts0W4POGWQZco05MJjFmoeIHs=
 github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5/go.mod h1:0Qr1uMHFmHsIYMcG4T7BJ9yrJtWadhOmpABCX69dwuc=
-github.com/belimawr/elastic-agent-libs v0.2.9-0.20231221105324-aedb70a4f832 h1:hCPNCDrtpZg8GekH7RptPcJ9C/Dgr2ebku2lETqFFw0=
-github.com/belimawr/elastic-agent-libs v0.2.9-0.20231221105324-aedb70a4f832/go.mod h1:EbRwBMsWoU4IHGKJlTrxbxC03hkihS9W4h+UgraLdDM=
+github.com/belimawr/elastic-agent-libs v0.2.9-0.20240116105334-25f61a14ad41 h1:4kwfzIBmNATT0es3HsgZP7W4p6OUo1TCOk5qchsUzTs=
+github.com/belimawr/elastic-agent-libs v0.2.9-0.20240116105334-25f61a14ad41/go.mod h1:pGMj5myawdqu+xE+WKvM5FQzKQ/MonikkWOzoFTJxaU=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/benbjohnson/immutable v0.2.1/go.mod h1:uc6OHo6PN2++n98KHLxW8ef4W42ylHiQSENghE1ezxI=
 github.com/benbjohnson/tmpl v1.0.0/go.mod h1:igT620JFIi44B6awvU9IsDhR77IXWtFigTLil/RPdps=

heartbeat/heartbeat.reference.yml

@@ -1636,6 +1636,42 @@ logging.files:
   # file. Defaults to true.
   # rotateonstartup: true
 
+# Some outputs will log raw events on errors like indexing errors in the
+# Elasticsearch output, to prevent logging raw events together with other
+# log messages, a different log file, only for log entries containing raw events,
+# is used. It will use the same level, selectors and all other configurations
+# from the default logger, but it will have it's own file configuration.
+#logging.events:
+  #files:
+    # Configure the path where the logs are written. The default is the logs directory
+    # under the home path (the binary location).
+    #path: /var/log/heartbeat
+
+    # The name of the files where the logs are written to.
+    #name: heartbeat-events-data
+
+    # Configure log file size limit. If the limit is reached, log file will be
+    # automatically rotated.
+    #rotateeverybytes: 10485760 # = 10MB
+
+    # Number of rotated log files to keep. The oldest files will be deleted first.
+    #keepfiles: 7
+
+    # The permissions mask to apply when rotating log files. The default value is 0600.
+    # Must be a valid Unix-style file permissions mask expressed in octal notation.
+    #permissions: 0600
+
+    # Enable log file rotation on time intervals in addition to the size-based rotation.
+    # Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h
+    # are boundary-aligned with minutes, hours, days, weeks, months, and years as
+    # reported by the local system clock. All other intervals are calculated from the
+    # Unix epoch. Defaults to disabled.
+    #interval: 0
+
+    # Rotate existing logs on startup rather than appending them to the existing
+    # file. Defaults to true.
+    # rotateonstartup: true
+
 # ============================= X-Pack Monitoring ==============================
 # Heartbeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

heartbeat/heartbeat.yml

@@ -152,6 +152,20 @@ processors:
 # "publisher", "service".
 #logging.selectors: ["*"]
 
+# Some outputs will log raw events on errors like indexing errors in the
+# Elasticsearch output, to prevent logging raw events together with other
+# log messages, a different log file, only for log entries containing raw events,
+# is used. It will use the same level, selectors and all other configurations
+# from the default logger, but it will have it's own file configuration.
+#logging.events:
+  #files:
+    # Configure the path where the logs are written. The default is the logs directory
+    # under the home path (the binary location).
+    #path: /var/log/heartbeat
+
+    # The name of the files where the logs are written to.
+    #name: heartbeat-events-data
+
 # ============================= X-Pack Monitoring ==============================
 # Heartbeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

libbeat/_meta/config/logging.reference.yml.tmpl

@@ -67,3 +67,39 @@ logging.files:
   # Rotate existing logs on startup rather than appending them to the existing
   # file. Defaults to true.
   # rotateonstartup: true
+
+# Some outputs will log raw events on errors like indexing errors in the
+# Elasticsearch output, to prevent logging raw events together with other
+# log messages, a different log file, only for log entries containing raw events,
+# is used. It will use the same level, selectors and all other configurations
+# from the default logger, but it will have it's own file configuration.
+#logging.events:
+  #files:
+    # Configure the path where the logs are written. The default is the logs directory
+    # under the home path (the binary location).
+    #path: /var/log/{{.BeatName}}
+
+    # The name of the files where the logs are written to.
+    #name: {{.BeatName}}-events-data
+
+    # Configure log file size limit. If the limit is reached, log file will be
+    # automatically rotated.
+    #rotateeverybytes: 10485760 # = 10MB
+
+    # Number of rotated log files to keep. The oldest files will be deleted first.
+    #keepfiles: 7
+
+    # The permissions mask to apply when rotating log files. The default value is 0600.
+    # Must be a valid Unix-style file permissions mask expressed in octal notation.
+    #permissions: 0600
+
+    # Enable log file rotation on time intervals in addition to the size-based rotation.
+    # Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h
+    # are boundary-aligned with minutes, hours, days, weeks, months, and years as
+    # reported by the local system clock. All other intervals are calculated from the
+    # Unix epoch. Defaults to disabled.
+    #interval: 0
+
+    # Rotate existing logs on startup rather than appending them to the existing
+    # file. Defaults to true.
+    # rotateonstartup: true

libbeat/_meta/config/logging.yml.tmpl

@@ -8,3 +8,17 @@
 # To enable all selectors, use ["*"]. Examples of other selectors are "beat",
 # "publisher", "service".
 #logging.selectors: ["*"]
+
+# Some outputs will log raw events on errors like indexing errors in the
+# Elasticsearch output, to prevent logging raw events together with other
+# log messages, a different log file, only for log entries containing raw events,
+# is used. It will use the same level, selectors and all other configurations
+# from the default logger, but it will have it's own file configuration.
+#logging.events:
+  #files:
+    # Configure the path where the logs are written. The default is the logs directory
+    # under the home path (the binary location).
+    #path: /var/log/{{.BeatName}}
+
+    # The name of the files where the logs are written to.
+    #name: {{.BeatName}}-events-data

libbeat/docs/loggingconfig.asciidoc

@@ -293,3 +293,73 @@ Below are some samples:
 `2017-12-17T18:54:16.242-0500	INFO	[example]	logp/core_test.go:16	some message`
 
 `2017-12-17T18:54:16.242-0500	INFO	[example]	logp/core_test.go:19	some message	{"x": 1}`
+
+ifndef::serverless[]
+[float]
+=== Configuration options for events logger
+
+Some outputs will log raw events on errors like indexing errors in the
+Elasticsearch output, to prevent logging raw events together with other
+log messages, a different log file, only for log entries containing raw events,
+is used. It will use the same level, selectors and all other configurations
+from the default logger, but it will have it's own file configuration.
+
+[float]
+==== `logging.events.files.path`
+
+The directory that log files are written to. The default is the logs path. See
+the <<directory-layout>> section for details.
+
+[float]
+==== `logging.events.files.name`
+
+The name of the file that logs are written to. The default is '{beatname_lc}'.
+
+[float]
+==== `logging.events.files.rotateeverybytes`
+
+The maximum size of a log file. If the limit is reached, a new log file is
+generated. The default size limit is 10485760 (10 MB).
+
+[float]
+==== `logging.events.files.keepfiles`
+
+The number of most recent rotated log files to keep on disk. Older files are
+deleted during log rotation. The default value is 7. The `keepfiles` options has
+to be in the range of 2 to 1024 files.
+
+[float]
+==== `logging.events.files.permissions`
+
+The permissions mask to apply when rotating log files. The default value is
+0600. The `permissions` option must be a valid Unix-style file permissions mask
+expressed in octal notation. In Go, numbers in octal notation must start with
+'0'.
+
+The most permissive mask allowed is 0640. If a higher permissions mask is
+specified via this setting, it will be subject to an umask of 0027.
+
+This option is not supported on Windows.
+
+Examples:
+
+* 0640: give read and write access to the file owner, and read access to members of the group associated with the file.
+* 0600: give read and write access to the file owner, and no access to all others.
+
+[float]
+==== `logging.events.files.interval`
+
+Enable log file rotation on time intervals in addition to size-based rotation.
+Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h
+are boundary-aligned with minutes, hours, days, weeks, months, and years as
+reported by the local system clock. All other intervals are calculated from the
+unix epoch. Defaults to disabled.
+
+[float]
+==== `logging.events.files.rotateonstartup`
+
+If the log file already exists on startup, immediately rotate it and start
+writing to a new file instead of appending to the existing one. Defaults to
+true.
+endif::serverless[]
+

libbeat/outputs/elasticsearch/elasticsearch.go

@@ -18,14 +18,15 @@
 package elasticsearch
 
 import (
+	"go.uber.org/zap"
+
 	"github.com/elastic/beats/v7/libbeat/beat"
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/libbeat/esleg/eslegclient"
 	"github.com/elastic/beats/v7/libbeat/outputs"
 	"github.com/elastic/beats/v7/libbeat/outputs/outil"
 	"github.com/elastic/elastic-agent-libs/config"
 	"github.com/elastic/elastic-agent-libs/logp"
-	"go.uber.org/zap"
 )
 
 func init() {

libbeat/outputs/fileout/file.go

@@ -23,14 +23,15 @@ import (
 	"path/filepath"
 	"time"
 
+	"go.uber.org/zap"
+
 	"github.com/elastic/beats/v7/libbeat/beat"
 	"github.com/elastic/beats/v7/libbeat/outputs"
 	"github.com/elastic/beats/v7/libbeat/outputs/codec"
 	"github.com/elastic/beats/v7/libbeat/publisher"
 	c "github.com/elastic/elastic-agent-libs/config"
 	"github.com/elastic/elastic-agent-libs/file"
 	"github.com/elastic/elastic-agent-libs/logp"
-	"go.uber.org/zap"
 )
 
 func init() {