From 57df903385a9bdbb1d7bb2ad35e30b4445da206d Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Wed, 1 Nov 2023 10:53:42 -0400 Subject: [PATCH] Update queue / ES connection defaults (#36990) --- CHANGELOG.next.asciidoc | 7 ++++++- libbeat/cmd/instance/beat_test.go | 4 ++-- libbeat/docs/queueconfig.asciidoc | 6 +++--- libbeat/outputs/elasticsearch/config.go | 12 ++++++++++-- .../elasticsearch/docs/elasticsearch.asciidoc | 4 ++-- libbeat/publisher/queue/memqueue/config.go | 7 ++++--- 6 files changed, 27 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 7d84cb5885f..35ff7048e39 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -11,7 +11,12 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] *Affecting all Beats* - The Elasticsearch output now enables compression by default. This decreases network data usage by an average of 70-80%, in exchange for 20-25% increased CPU use and ~10% increased ingestion time. The previous default can be restored by setting the flag `compression_level: 0` under `output.elasticsearch`. {pull}36681[36681] - Elastic-agent-autodiscover library updated to version 0.6.4, disabling metadata for deployment and cronjob. Pods that will be created from deployments or cronjobs will not have the extra metadata field for kubernetes.deployment or kubernetes.cronjob, respectively. {pull}36877[36877] - +- Defaults are changing for some options in the queue and Elasticsearch output, to improve typical performance based on current benchmark data. All changes can be overridden by setting them explicitly in the beat configuration. {pull}36990[36990] The changes are: + - `queue.mem.events` is changing from `4096` to `3200`. + - `queue.mem.flush.min_events` is changing from `2048` to `1600`. + - `queue.mem.flush.timeout` is changing from `1s` to `10s`. + - `output.elasticsearch.bulk_max_size` is changing from `50` to `1600`. + - `output.elasticsearch.idle_connection_timeout` is changing from `60s` to `3s`. *Auditbeat* diff --git a/libbeat/cmd/instance/beat_test.go b/libbeat/cmd/instance/beat_test.go index 03474ecfcd9..52e55941225 100644 --- a/libbeat/cmd/instance/beat_test.go +++ b/libbeat/cmd/instance/beat_test.go @@ -279,7 +279,7 @@ func TestPromoteOutputQueueSettings(t *testing.T) { }{ "blank": { input: []byte(""), - memEvents: 4096, + memEvents: 3200, }, "defaults": { input: []byte(` @@ -289,7 +289,7 @@ output: hosts: - "localhost:9200" `), - memEvents: 4096, + memEvents: 3200, }, "topLevelQueue": { input: []byte(` diff --git a/libbeat/docs/queueconfig.asciidoc b/libbeat/docs/queueconfig.asciidoc index ade3bd2ec8e..8fd4bed0416 100644 --- a/libbeat/docs/queueconfig.asciidoc +++ b/libbeat/docs/queueconfig.asciidoc @@ -61,7 +61,7 @@ You can specify the following options in the `queue.mem` section of the +{beatna Number of events the queue can store. -The default value is 4096 events. +The default value is 3200 events. [float] ===== `flush.min_events` @@ -70,7 +70,7 @@ Minimum number of events required for publishing. If this value is set to 0, the output can start publishing events without additional waiting times. Otherwise the output has to wait for more events to become available. -The default value is 2048. +The default value is 1600. [float] ===== `flush.timeout` @@ -78,7 +78,7 @@ The default value is 2048. Maximum wait time for `flush.min_events` to be fulfilled. If set to 0s, events will be immediately available for consumption. -The default value is 1s. +The default value is 10s. [float] [[configuration-internal-queue-disk]] diff --git a/libbeat/outputs/elasticsearch/config.go b/libbeat/outputs/elasticsearch/config.go index e504f2dc213..6d8016b9636 100644 --- a/libbeat/outputs/elasticsearch/config.go +++ b/libbeat/outputs/elasticsearch/config.go @@ -54,7 +54,7 @@ type Backoff struct { } const ( - defaultBulkSize = 50 + defaultBulkSize = 1600 ) var ( @@ -74,10 +74,18 @@ var ( Init: 1 * time.Second, Max: 60 * time.Second, }, - Transport: httpcommon.DefaultHTTPTransportSettings(), + Transport: esDefaultTransportSettings(), } ) +func esDefaultTransportSettings() httpcommon.HTTPTransportSettings { + transport := httpcommon.DefaultHTTPTransportSettings() + // The ES output differs from the common transport settings by having + // a 3-second idle timeout + transport.IdleConnTimeout = 3 * time.Second + return transport +} + func (c *elasticsearchConfig) Validate() error { if c.APIKey != "" && (c.Username != "" || c.Password != "") { return fmt.Errorf("cannot set both api_key and username/password") diff --git a/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc b/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc index 6af56ac42db..007bf92355d 100644 --- a/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc +++ b/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc @@ -661,7 +661,7 @@ endif::[] ===== `bulk_max_size` -The maximum number of events to bulk in a single Elasticsearch bulk API index request. The default is 50. +The maximum number of events to bulk in a single Elasticsearch bulk API index request. The default is 1600. Events can be collected into batches. {beatname_uc} will split batches larger than `bulk_max_size` into multiple batches. @@ -693,7 +693,7 @@ Elasticsearch after a network error. The default is `60s`. The maximum amount of time an idle connection will remain idle before closing itself. Zero means no limit. The format is a Go language duration (example 60s is 60 seconds). -The default is 0. +The default is 3s. ===== `timeout` diff --git a/libbeat/publisher/queue/memqueue/config.go b/libbeat/publisher/queue/memqueue/config.go index 9e3be0a7611..5e4f78ae41c 100644 --- a/libbeat/publisher/queue/memqueue/config.go +++ b/libbeat/publisher/queue/memqueue/config.go @@ -32,9 +32,9 @@ type config struct { } var defaultConfig = config{ - Events: 4 * 1024, - FlushMinEvents: 2 * 1024, - FlushTimeout: 1 * time.Second, + Events: 3200, + FlushMinEvents: 1600, + FlushTimeout: 10 * time.Second, } func (c *config) Validate() error { @@ -53,6 +53,7 @@ func SettingsForUserConfig(cfg *c.C) (Settings, error) { return Settings{}, fmt.Errorf("couldn't unpack memory queue config: %w", err) } } + //nolint:gosimple // Actually want this conversion to be explicit since the types aren't definitionally equal. return Settings{ Events: config.Events, FlushMinEvents: config.FlushMinEvents,