From f7f4a1f36d32aee2ce2e364a0aa1564474df6d6d Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Thu, 30 Jan 2025 19:10:44 -0500 Subject: [PATCH 01/10] Update Journald fields to better match ECS (#42403) The fields produced by the Journald input are updated to better match ECS. Renamed fields: Dropped fields: `syslog.priority` and `syslog.facility` while keeping their duplicated equivalent: `log.syslog.priority`,`log.syslog.facility.code`. Renamed fields: `syslog.identifier` -> `log.syslog.appname`, `syslog.pid` -> `log.syslog.procid`. `container.id_truncated` is dropped because the full container ID is already present as `container.id` and `container.log.tag` is dropped because it is already present as `log.syslog.appname`. The field `container.partial` is replaced by the tag `partial_message` if it was `true`, otherwise no tag is added. --- CHANGELOG.next.asciidoc | 7 +++ filebeat/docs/inputs/input-journald.asciidoc | 14 ++--- filebeat/input/journald/input.go | 15 ++++++ .../input/journald/input_filtering_test.go | 4 +- filebeat/input/journald/input_parsers_test.go | 30 +++++++++++ filebeat/input/journald/input_test.go | 6 +++ .../journald/pkg/journalfield/conv_test.go | 21 ++++---- .../journald/pkg/journalfield/default.go | 12 ++--- .../input-multiline-parser-events.json | 50 ++++-------------- .../journald/testdata/ndjson-parser.export | Bin 830 -> 861 bytes .../testdata/ndjson-parser.journal.gz | Bin 10227 -> 10320 bytes .../auth/test/debian-12.journal-expected.json | 16 ++++++ .../test/debian-12.journal-expected.json | 3 ++ 13 files changed, 113 insertions(+), 65 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 91e75266659d..96576690f1e4 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -59,6 +59,13 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Filestream inputs can define `allow_deprecated_id_duplication: true` to run keep the previous behaviour of running inputs with duplicated IDs. {issue}41938[41938] {pull}41954[41954] - The Filestream input only starts to ingest a file when it is >= 1024 bytes in size. This happens because the fingerprint` is the default file identity now. To restore the previous behaviour, set `file_identity.native: ~` and `prospector.scanner.fingerprint.enabled: false` {issue}40197[40197] {pull}41762[41762] - Filebeat fails to start when its configuration contains usage of the deprecated `log` or `container` inputs. However, they can still be using while setting `allow_deprecated_use: true` in their configuration {pull}42295[42295] +- The fields produced by the Journald input are updated to better match ECS. Renamed fields: +Dropped fields: `syslog.priority` and `syslog.facility` while keeping their duplicated equivalent: +`log.syslog.priority`,`log.syslog.facility.code`. Renamed fields: `syslog.identifier` -> `log.syslog.appname`, +`syslog.pid` -> `log.syslog.procid`. `container.id_truncated` is dropped because the full container ID is +already present as `container.id` and `container.log.tag` is dropped because it is already present as +`log.syslog.appname`. The field `container.partial` is replaced by the tag `partial_message` if it was `true`, +otherwise no tag is added. {issue}42208[42208] {pull}42403[42403] *Heartbeat* diff --git a/filebeat/docs/inputs/input-journald.asciidoc b/filebeat/docs/inputs/input-journald.asciidoc index f7655f51419f..d2bf341d60f0 100644 --- a/filebeat/docs/inputs/input-journald.asciidoc +++ b/filebeat/docs/inputs/input-journald.asciidoc @@ -461,10 +461,10 @@ journald fields: `_MACHINE_ID`:: `host.id` `_MESSAGE`:: `message` `_PID`:: `process.pid` -`_PRIORITY`:: `syslog.priority` -`_SYSLOG_FACILITY`:: `syslog.facility` -`_SYSLOG_IDENTIFIER`:: `syslog.identifier` -`_SYSLOG_PID`:: `syslog.pid` +`_PRIORITY`:: `log.syslog.priority` +`_SYSLOG_FACILITY`:: `log.syslog.facility.code` +`_SYSLOG_IDENTIFIER`:: `log.syslog.appname` +`_SYSLOG_PID`:: `log.syslog.procid` `_SYSTEMD_CGROUP`:: `systemd.cgroup` `_SYSTEMD_INVOCATION_ID`:: `systemd.invocation_id` `_SYSTEMD_OWNER_UID`:: `systemd.owner_uid` @@ -484,13 +484,13 @@ https://docs.docker.com/config/containers/logging/journald/[Docker] are also available: [horizontal] -`CONTAINER_ID`:: `container.id_truncated` `CONTAINER_ID_FULL`:: `container.id` `CONTAINER_NAME`:: `container.name` -`CONTAINER_PARTIAL_MESSAGE`:: `container.partial` -`CONTAINER_TAG`:: `container.log.tag` `IMAGE_NAME`:: `container.image.name` +If `CONTAINER_PARTIAL_MESSAGE` is present and it is true, then the tag +`partial_message` is added to the final event. + [id="{beatname_lc}-input-{type}-common-options"] include::../inputs/input-common-options.asciidoc[] diff --git a/filebeat/input/journald/input.go b/filebeat/input/journald/input.go index 0ab3c5481775..e8eb8a7a8773 100644 --- a/filebeat/input/journald/input.go +++ b/filebeat/input/journald/input.go @@ -22,6 +22,7 @@ package journald import ( "errors" "fmt" + "strconv" "time" "github.com/elastic/beats/v7/filebeat/input/journald/pkg/journalctl" @@ -279,6 +280,20 @@ func (r *readerAdapter) Next() (reader.Message, error) { fields.Put("event.kind", "event") fields.Put("event.created", created) + // IF 'container.partial' is present, we can parse it and it's true, then + // add 'partial_message' to tags. + if partialMessageRaw, err := fields.GetValue("container.partial"); err == nil { + partialMessage, err := strconv.ParseBool(fmt.Sprint(partialMessageRaw)) + if err == nil && partialMessage { + // 'fields' came directly from the journal, + // so there is no chance tags already exist + fields.Put("tags", []string{"partial_message"}) + } + } + + // Delete 'container.partial', if there are any errors, ignore it + _ = fields.Delete("container.partial") + // if entry is coming from a remote journal, add_host_metadata overwrites // the source hostname, so it has to be copied to a different field if r.saveRemoteHostname { diff --git a/filebeat/input/journald/input_filtering_test.go b/filebeat/input/journald/input_filtering_test.go index 9464016d40dd..34d0755393bd 100644 --- a/filebeat/input/journald/input_filtering_test.go +++ b/filebeat/input/journald/input_filtering_test.go @@ -142,7 +142,7 @@ func TestInputIncludeMatches(t *testing.T) { "single match condition": { includeMatches: map[string]interface{}{ "match": []string{ - "syslog.facility=3", + "log.syslog.facility.code=3", }, }, expectedMessages: []string{ @@ -159,7 +159,7 @@ func TestInputIncludeMatches(t *testing.T) { includeMatches: map[string]interface{}{ "match": []string{ "journald.process.name=systemd", - "syslog.facility=3", + "log.syslog.facility.code=3", }, }, expectedMessages: []string{ diff --git a/filebeat/input/journald/input_parsers_test.go b/filebeat/input/journald/input_parsers_test.go index 35df3cc1e848..de9becc386f2 100644 --- a/filebeat/input/journald/input_parsers_test.go +++ b/filebeat/input/journald/input_parsers_test.go @@ -31,6 +31,9 @@ import ( // it only tests a single parser, but that is enough to ensure // we're correctly using the parsers func TestInputParsers(t *testing.T) { + // If this test fails, uncomment the lopg setup line + // to send logs to stderr + // logp.DevelopmentSetup() out := decompress(t, filepath.Join("testdata", "ndjson-parser.journal.gz")) env := newInputTestingEnvironment(t) @@ -71,3 +74,30 @@ func TestInputParsers(t *testing.T) { t.Errorf("expecting 'answer' from the Journal JSON to be '%d' got '%d' instead", expectedAnswer, answer) } } + +func TestPartialMessageTag(t *testing.T) { + out := decompress(t, filepath.Join("testdata", "ndjson-parser.journal.gz")) + env := newInputTestingEnvironment(t) + inp := env.mustCreateInput(mapstr.M{ + "paths": []string{out}, + }) + + ctx, cancelInput := context.WithCancel(context.Background()) + t.Cleanup(cancelInput) + env.startInput(ctx, inp) + env.waitUntilEventCount(1) + event := env.pipeline.clients[0].GetEvents()[0] + + tags, err := event.Fields.GetValue("tags") + if err != nil { + t.Fatalf("'tags' not found in event: %s", err) + } + + tagsStrSlice, ok := tags.([]string) + if !ok { + t.Fatalf("expecting 'tags' to be []string, got %T instead", tags) + } + if tagsStrSlice[0] != "partial_message" { + t.Fatalf("expecting the tag 'partial_message', got %v instead", tagsStrSlice) + } +} diff --git a/filebeat/input/journald/input_test.go b/filebeat/input/journald/input_test.go index 776115d5d8ac..5d0edb18cb05 100644 --- a/filebeat/input/journald/input_test.go +++ b/filebeat/input/journald/input_test.go @@ -119,6 +119,12 @@ func TestInputFieldsTranslation(t *testing.T) { // events in the same format as the original one. We use the events from the // already existing journal file 'input-multiline-parser.journal' // +// Generating golden file: to generate the golden file you need to copy +// and run this test on a older version that still uses go-systemd, +// like 8.16.0, so the input run on this older version, call +// `env.pipeline.GetAllEvents()`, get the events, marshal them as +// JSON with " " as the indent argument and write it to the file. +// // The following fields are not currently tested: // __CURSOR - it is added to the registry and there are other tests for it // __MONOTONIC_TIMESTAMP - it is part of the cursor diff --git a/filebeat/input/journald/pkg/journalfield/conv_test.go b/filebeat/input/journald/pkg/journalfield/conv_test.go index c92d3f15bcb0..936b7ce485ee 100644 --- a/filebeat/input/journald/pkg/journalfield/conv_test.go +++ b/filebeat/input/journald/pkg/journalfield/conv_test.go @@ -50,8 +50,10 @@ func TestConversion(t *testing.T) { "SYSLOG_PID": "123456", }, want: mapstr.M{ - "syslog": mapstr.M{ - "pid": int64(123456), + "log": mapstr.M{ + "syslog": mapstr.M{ + "procid": int64(123456), + }, }, }, }, @@ -60,9 +62,6 @@ func TestConversion(t *testing.T) { "PRIORITY": "123456, ", }, want: mapstr.M{ - "syslog": mapstr.M{ - "priority": int64(123456), - }, "log": mapstr.M{ "syslog": mapstr.M{ "priority": int64(123456), @@ -75,8 +74,10 @@ func TestConversion(t *testing.T) { "SYSLOG_PID": "123456,root", }, want: mapstr.M{ - "syslog": mapstr.M{ - "pid": int64(123456), + "log": mapstr.M{ + "syslog": mapstr.M{ + "procid": int64(123456), + }, }, }, }, @@ -85,8 +86,10 @@ func TestConversion(t *testing.T) { "SYSLOG_PID": "", }, want: mapstr.M{ - "syslog": mapstr.M{ - "pid": "", + "log": mapstr.M{ + "syslog": mapstr.M{ + "procid": "", + }, }, }, }, diff --git a/filebeat/input/journald/pkg/journalfield/default.go b/filebeat/input/journald/pkg/journalfield/default.go index d2ec76fcd393..bc508750765b 100644 --- a/filebeat/input/journald/pkg/journalfield/default.go +++ b/filebeat/input/journald/pkg/journalfield/default.go @@ -38,10 +38,10 @@ var journaldEventFields = FieldConversion{ "OBJECT_SYSTEMD_UNIT": text("journald.object.systemd.unit"), "OBJECT_SYSTEMD_USER_UNIT": text("journald.object.systemd.user_unit"), "OBJECT_UID": integer("journald.object.uid"), - "PRIORITY": integer("syslog.priority", "log.syslog.priority"), - "SYSLOG_FACILITY": integer("syslog.facility", "log.syslog.facility.code"), - "SYSLOG_IDENTIFIER": text("syslog.identifier"), - "SYSLOG_PID": integer("syslog.pid"), + "PRIORITY": integer("log.syslog.priority"), + "SYSLOG_FACILITY": integer("log.syslog.facility.code"), + "SYSLOG_IDENTIFIER": text("log.syslog.appname"), + "SYSLOG_PID": integer("log.syslog.procid"), "UNIT": text("journald.unit"), "_AUDIT_LOGINUID": integer("journald.audit.login_uid"), "_AUDIT_SESSION": text("journald.audit.session"), @@ -74,14 +74,14 @@ var journaldEventFields = FieldConversion{ "_UID": integer("journald.uid"), // docker journald fields from: https://docs.docker.com/config/containers/logging/journald/ - "CONTAINER_ID": text("container.id_truncated"), "CONTAINER_ID_FULL": text("container.id"), "CONTAINER_NAME": text("container.name"), - "CONTAINER_TAG": text("container.log.tag"), "CONTAINER_PARTIAL_MESSAGE": text("container.partial"), "IMAGE_NAME": text("container.image.name"), // dropped fields + "CONTAINER_ID": ignoredField, + "CONTAINER_TAG": ignoredField, "_SOURCE_MONOTONIC_TIMESTAMP": ignoredField, // saved in the registry "_SOURCE_REALTIME_TIMESTAMP": ignoredField, // saved in the registry "__CURSOR": ignoredField, // saved in the registry diff --git a/filebeat/input/journald/testdata/input-multiline-parser-events.json b/filebeat/input/journald/testdata/input-multiline-parser-events.json index 0d8fbff7f58d..166d97bfa3a5 100644 --- a/filebeat/input/journald/testdata/input-multiline-parser-events.json +++ b/filebeat/input/journald/testdata/input-multiline-parser-events.json @@ -33,6 +33,7 @@ }, "log": { "syslog": { + "appname": "sudo", "facility": { "code": 10 }, @@ -98,11 +99,6 @@ } } }, - "syslog": { - "facility": 10, - "identifier": "sudo", - "priority": 6 - }, "systemd": { "cgroup": "/user.slice/user-1000.slice/session-1.scope", "invocation_id": "2172b101bd684f7b9ca1c434ff2cdd80", @@ -165,8 +161,10 @@ }, "uid": 1000 }, + "message_id": "39f53479d3a045ac8e11786248231fbf", "log": { "syslog": { + "appname": "systemd", "facility": { "code": 3 }, @@ -174,7 +172,6 @@ } }, "message": "Started Outputs some log lines.", - "message_id": "39f53479d3a045ac8e11786248231fbf", "process": { "args": [ "/usr/lib/systemd/systemd", @@ -184,11 +181,6 @@ "command_line": "/usr/lib/systemd/systemd --user", "pid": 1003 }, - "syslog": { - "facility": 3, - "identifier": "systemd", - "priority": 6 - }, "systemd": { "cgroup": "/user.slice/user-1000.slice/user@1000.service/init.scope", "owner_uid": 1000, @@ -240,6 +232,7 @@ }, "log": { "syslog": { + "appname": "cat", "facility": { "code": 3 }, @@ -250,11 +243,6 @@ "process": { "pid": 2084785 }, - "syslog": { - "facility": 3, - "identifier": "cat", - "priority": 6 - }, "systemd": { "cgroup": "/user.slice/user-1000.slice/user@1000.service/app.slice/log-service.service", "invocation_id": "ceb98539accb4f60be76734b4ddd6d44", @@ -307,6 +295,7 @@ }, "log": { "syslog": { + "appname": "cat", "facility": { "code": 3 }, @@ -317,11 +306,6 @@ "process": { "pid": 2084785 }, - "syslog": { - "facility": 3, - "identifier": "cat", - "priority": 6 - }, "systemd": { "cgroup": "/user.slice/user-1000.slice/user@1000.service/app.slice/log-service.service", "invocation_id": "ceb98539accb4f60be76734b4ddd6d44", @@ -374,6 +358,7 @@ }, "log": { "syslog": { + "appname": "cat", "facility": { "code": 3 }, @@ -384,11 +369,6 @@ "process": { "pid": 2084785 }, - "syslog": { - "facility": 3, - "identifier": "cat", - "priority": 6 - }, "systemd": { "cgroup": "/user.slice/user-1000.slice/user@1000.service/app.slice/log-service.service", "invocation_id": "ceb98539accb4f60be76734b4ddd6d44", @@ -441,6 +421,7 @@ }, "log": { "syslog": { + "appname": "cat", "facility": { "code": 3 }, @@ -451,11 +432,6 @@ "process": { "pid": 2084785 }, - "syslog": { - "facility": 3, - "identifier": "cat", - "priority": 6 - }, "systemd": { "cgroup": "/user.slice/user-1000.slice/user@1000.service/app.slice/log-service.service", "invocation_id": "ceb98539accb4f60be76734b4ddd6d44", @@ -508,6 +484,7 @@ }, "log": { "syslog": { + "appname": "cat", "facility": { "code": 3 }, @@ -518,11 +495,6 @@ "process": { "pid": 2084785 }, - "syslog": { - "facility": 3, - "identifier": "cat", - "priority": 6 - }, "systemd": { "cgroup": "/user.slice/user-1000.slice/user@1000.service/app.slice/log-service.service", "invocation_id": "ceb98539accb4f60be76734b4ddd6d44", @@ -575,6 +547,7 @@ }, "log": { "syslog": { + "appname": "cat", "facility": { "code": 3 }, @@ -585,11 +558,6 @@ "process": { "pid": 2084785 }, - "syslog": { - "facility": 3, - "identifier": "cat", - "priority": 6 - }, "systemd": { "cgroup": "/user.slice/user-1000.slice/user@1000.service/app.slice/log-service.service", "invocation_id": "ceb98539accb4f60be76734b4ddd6d44", diff --git a/filebeat/input/journald/testdata/ndjson-parser.export b/filebeat/input/journald/testdata/ndjson-parser.export index 0a24b593f776c6d9855902cdb759b7e578756830..a85e04ceb86bfeeda42f83e91dacafb033cf82ac 100644 GIT binary patch delta 61 zcmdnTc9(5~2D7QNzh8)>r=M$3e1Kz6h^M1Zysv9;u%o-HZAnpSDp!1fr;DwjnW32( QSG=>oudi)#VhI-)07lml%>V!Z delta 30 lcmcc1wvTOt2D4avfTxSCp_!qX8CSfszpt-ta$*S=7XXRu2s8iy diff --git a/filebeat/input/journald/testdata/ndjson-parser.journal.gz b/filebeat/input/journald/testdata/ndjson-parser.journal.gz index f5eb86364da58dd9447790e74fd3f040dea6fa2d..a279f6238bca2b381538fa9339d5d101287779ce 100644 GIT binary patch literal 10320 zcmeI1X;72b8ppX^L~ARx(4qxRU08$yDk?&N+_6r*z!j=sp+Jbprfd-cM3y9WL97vS z4I*0#BM32I2#A7&Kq_g2!331;We+KV2nh)gLP*{$*UsEeUOw3A+!^f6`|X@L&w2mP z@0|00&O0vzzj0&F&xJ8YZzo2^qL7IW7sHcLQOS<6$kgP-@OUH15GFo!pxF2lYBi%R$V(O;Ee;bB&Fzqc|q z$W?tysbSIWIZC1Wv_$Yk1q*6&jt|B}f^>r>yP;yx;jni7bbFytKJ?xHbZ2PvDfNzPoti(b z-7vvlm;l!bYVzM^_@9K&CJqHykYDF3;xZ5o?pxCn|HACvfYc@<%^a@!Z!+2s3|P0g za$sZiFE|#Z$kooS9ymHytubXQ5COJkpxl1+X>soNRksFj{WCN%;Q}vm!-j)N)l|wp z|D9L6a^I$ML?Wx4r{;6Qd`p$m2D@sagl`*3SBU#u(i2KPwP0n1>rAZD zg;jiqM{^;@<(sCOY~+5qrEnWAZZ?707BkP{t43=?2nAWhq-KpW@LN1Qw3xZh%Gsnw zaf*$K?}5&xg~C$&2ju9;RuE;5Ac<{ffSC~smWI=F$~i;$q?ex+D`L;eATuu~rl8Y_ zXqu~n5c4IPL7wx>K*cS=F=93JJ@~I<^mE|hkPSIa@neiChJY50D758l-U3^Ab}}a| z`$*g;JM!|rvFUfRZke3u!!BrO6HCQ4VAl$jYYqZtU!VN?Cxw4Dc7&LPU{&HbM+7ns zU*~UEdlAa(GQS}PW}Va6*Lsr+Yni@~S%(?7z!h^xahHNS)^@vGPRtU52siQVlf}sF zMf<$to!iUnn4Y2J6WC|C&OGpomN{?^o-cdy5aAp;k=7j$@$?F9#6762B}`xps+@Gw zWF;bL^2S#&h$x&BP)yN3-wXIMS{2L*my11{lmh@%l}b2*_B~${T}_kcOVo<(J~ApO zx&Kv+*ys7;+?|;gJB|}=$gN5FZ8x+=N^>4FD&s+ z_`dj7Zf;d_1gg5>P%D3CuiPAt(eV>95^;A!U|Y`&p5y`f!Wr0@{KsaE_Ke_2mcld8 zJOd9O?hXaJmVTb;y;aS9&dkAB~VzX?@4(y-zNk$6DC%4 znM|uKl5rKvG&4GCo&}f?Zqn+~r!i?yhn%UcgKaqEHYVNLlf7C@53dYa9++_A!o-|o z16s4p`|!5$?6Py!%WnMU2nyZm;P{+H!czPY-1XL+54diY@`r?(V_ z6CWSYTEfv8sJ1rMvA|5Tx)ak1PMfHi>8UR??E^6W30c<{ol{wlr{s`Y<{dGAr+a8B zOrV+#8t{K1PK2RjvyZAR%R(B0NYK!oJ15*9Oq0&5?h78@rcMuIauiJ$Rn_K*Rlo-O z>5S_(wQrBJ2M$SW=4ewRGs^=BF1Xblsz!qny6cj zDpfnFrJIy}KtVZUAS|is7((EV_7dv|^BLI5%-h}9REzdvZVfJhSPrsP&7ULNm!+ms zd;Y_wXEz*tu>%w-$?<#+X3iT<8Dh={C~SfpH*#OJzA6)Lzv289;rOah21asWE6&k#~3Sb=|RGiCc7(-2cRn(s;iES_tId@+@PbzJ_TThs~2{O(mJA>D)m9WKegqGip)MaqY=tX2CK!M z%`PZ43onkXXt+Ulw$^Kx1yH6Gs_i5LbFtEBo~(I2uaq_a*$Npk?u$e=}_$>S#-EsNk#>v z4_A2gemNL>Nq#tNBNz}wV5uR4kLz_;QmyZ03S75{y)Gqope&@L3ydE?tFc0XauL!*&t~k5a8$Zh}aDc z8pOz!4AQd177&CWghWN55!s|eL?97a0)!CALdZrY?Nm+m`|DP{bH4j-y;tuHhsr_# zoBSO+o>fF8Y&FJROCsWN&dKov;&p;c5 z?7Fef6KchpM+f1a!>==+`CPwqBeB!%5EWZx?QaA+uC2y;I;iwqke%N51ph(lwMbR)Ng!(__hRY4 z^@M-SegB=+c1PH0xdcHO3UUZ^bhVngWy=GHuQxhdW z3FKhPF~OS7ah&>qe-z% z!LtSS?BNbPL^yJG=1X{U!qh`K5{gUXhxu`6a_)XdRuTF2hAq`ctTQf-Hx_~}FGnHv z*6u5wHgW^qe1;u)@b?l!9SdS{oHCEX?ONiE?TtVZ=LBkR{Wimf9q6X@Q=0i#gA3?$ zS5kIH1-vDGdyA3f&8gNai2Lv@e_e-8C!+7eTdKZ+Zw1{9bEm#FK;n(-cYM}~?*?5s zW@Nzo8G(9!K4~ z>45re29AZ11&QfAap>{B|9fvvwP5`3!wsfr!<={Q-+Ue6==OD1q2U4y{!&mAwCwo_ zxSc@~2wMK7@X4L8Jsftt)bo5|s=uBo2*es4!xV=7+1v-{epg&UpOpb|n)3I3di9kx z;iZV4>OzqxyURx~O7QbP;?~H4#*`1d$^C@E+K=+6Ke;K~-AnyY;dLudf_z~veM`1B zhnWOHgiF6^1AiY@LmsiZz`#8e&e{SUS_-5F{D(HOUL;z;BuX%SKz$WXu0lzC7an() z6=eda3d)}!f&XHX8cmr@?rpz8UrwC~WW%QXl{GaT9UTkmTF489RY~BGS`B(|E2W>J z%Ig(JLf+F$-IX6+F@B~|gs?@i5Vodz@U6uVlKAp>1-{_Bp2sU!DQF8^)Vd-DPh!lf z$}PP!2@u*t^gCun8M?-vI5L!IJ=ImxeBDwADASo=)&PZZXy_M%zpMZQWCD!dysQkUwd z%c5u+%E#O1o1Va;q#c%>AvgMz?{Toix}`m+i;*GO6g>+k+OL`9y1N#jfW2eIU|3js zFvTn!qnaq;0qB%Pmqk zq_y;M&Rlsd$ni*}^0@Auj~#eTr2izpDwxh4IP8JX0pc~fT|-<(K$)X`gSS*2nh><6 zZsU%}>HE*61(T;kfQz*PR7<~TX~nW*5^4cG>XeixStFhUd(R=zdfk`hJip4bF^?*_ z=LU#zlssPChO(?EjtTUM&{#|oF0tPfKhj04u4CFWewk5wl-Ik;$K+$mgk~9oY{G3h z$?fuPNWVnd#Icu_v%xVZh}VFRmIrx31H95%oi#)NY!oR1S6nhG2gFEKThCZ*_s(3l zs>>wl^4DeONwjE~;!DIoB|H!5WS*j+AcSWsD$RI=>pU%h*_E;qq`VcK|rNfNHNh{waqn38bq=tT|-WtK&oJrWN^_qq1~6 z&y>+VN7qq*iwZS*0d{t-nio42AM zuE1CSq6!vQp$iSqnMiru=q)HkW31!QI@V@8+q{>kQ9r_9gHWipyH6Vi#*~*L5@s;5 z&?QZl?kLnb6AO$$z~3$w_4J-i9hK!;S@KfTT{2eNx&x63qT!iQQdq_e)eWFaqyvAmwdCetF^U? z!z6Jk_bz2`(nG+~$~^p@883?0M33q9vm{jiXeuhQ#B;+nApF7E|HY;fQI6{iA=^i^ zCVjeBoWO5Mn``B4i1*X8&JZte_#DWg<_aLw>7g54<~h_`n?Bm$9l9SDCj4!z)WXQ& zfZJ?$c}f+2cQ#^y=f~Nn{&m>w7TdO>cUo&EjQ0S#GY$-LlEJrRkmZ(my_qKVV?_7n zzV+7C&D%O#md&(}>Cjb@&1>0STK8&puJBRw4Z+m7%*c>Y3FgJCMhDIGXtO;#|ICnW ZIzVh{Y={=H(XM23sT+M@%a)s4{tNd`$0h&( diff --git a/filebeat/module/system/auth/test/debian-12.journal-expected.json b/filebeat/module/system/auth/test/debian-12.journal-expected.json index 2ef69b76b22a..21648f491b58 100644 --- a/filebeat/module/system/auth/test/debian-12.journal-expected.json +++ b/filebeat/module/system/auth/test/debian-12.journal-expected.json @@ -17,8 +17,10 @@ "host.hostname": "vagrant-debian-12", "host.id": "5e6dc8fe417f4ea383e2afaa731f5d8a", "input.type": "journald", + "log.syslog.appname": "sshd", "log.syslog.facility.code": 4, "log.syslog.priority": 6, + "log.syslog.procid": 26538, "message": "Accepted publickey for vagrant from 10.0.2.2 port 48274 ssh2: ED25519 SHA256:k1kjhwoH/H3w31MbGOIGd7qxrkSQJnoAN0eYJVHDmmI", "process.args": [ "\"sshd: vagrant [priv]\"" @@ -65,8 +67,10 @@ "host.hostname": "vagrant-debian-12", "host.id": "5e6dc8fe417f4ea383e2afaa731f5d8a", "input.type": "journald", + "log.syslog.appname": "sshd", "log.syslog.facility.code": 4, "log.syslog.priority": 6, + "log.syslog.procid": 1710, "message": "Accepted password for vagrant from 192.168.42.119 port 55310 ssh2", "process.args": [ "\"sshd: vagrant [priv]\"" @@ -111,8 +115,10 @@ "host.hostname": "vagrant-debian-12", "host.id": "5e6dc8fe417f4ea383e2afaa731f5d8a", "input.type": "journald", + "log.syslog.appname": "sshd", "log.syslog.facility.code": 4, "log.syslog.priority": 6, + "log.syslog.procid": 1721, "message": "Invalid user test from 192.168.42.119 port 48890", "process.args": [ "\"sshd: unknown [priv]\"" @@ -155,8 +161,10 @@ "host.hostname": "vagrant-debian-12", "host.id": "5e6dc8fe417f4ea383e2afaa731f5d8a", "input.type": "journald", + "log.syslog.appname": "sshd", "log.syslog.facility.code": 4, "log.syslog.priority": 6, + "log.syslog.procid": 1723, "message": "Failed password for root from 192.168.42.119 port 46632 ssh2", "process.args": [ "\"sshd: root [priv]\"" @@ -201,8 +209,10 @@ "host.hostname": "vagrant-debian-12", "host.id": "5e6dc8fe417f4ea383e2afaa731f5d8a", "input.type": "journald", + "log.syslog.appname": "sshd", "log.syslog.facility.code": 4, "log.syslog.priority": 6, + "log.syslog.procid": 1723, "message": "Failed password for root from 192.168.42.119 port 46632 ssh2", "process.args": [ "\"sshd: root [priv]\"" @@ -247,8 +257,10 @@ "host.hostname": "vagrant-debian-12", "host.id": "5e6dc8fe417f4ea383e2afaa731f5d8a", "input.type": "journald", + "log.syslog.appname": "sshd", "log.syslog.facility.code": 4, "log.syslog.priority": 6, + "log.syslog.procid": 1723, "message": "Failed password for root from 192.168.42.119 port 46632 ssh2", "process.args": [ "\"sshd: root [priv]\"" @@ -285,6 +297,7 @@ "host.hostname": "vagrant-debian-12", "host.id": "5e6dc8fe417f4ea383e2afaa731f5d8a", "input.type": "journald", + "log.syslog.appname": "sudo", "log.syslog.facility.code": 10, "log.syslog.priority": 5, "message": " vagrant : TTY=pts/2 ; PWD=/home/vagrant ; USER=root ; COMMAND=/usr/bin/emacs /etc/ssh/sshd_config", @@ -333,8 +346,10 @@ "host.hostname": "vagrant-debian-12", "host.id": "5e6dc8fe417f4ea383e2afaa731f5d8a", "input.type": "journald", + "log.syslog.appname": "groupadd", "log.syslog.facility.code": 10, "log.syslog.priority": 6, + "log.syslog.procid": 1743, "message": "new group: name=test, GID=1001", "process.args": [ "/sbin/groupadd", @@ -363,6 +378,7 @@ "host.hostname": "vagrant-debian-12", "host.id": "5e6dc8fe417f4ea383e2afaa731f5d8a", "input.type": "journald", + "log.syslog.appname": "systemd-logind", "log.syslog.facility.code": 4, "log.syslog.priority": 6, "message": "Session 8 logged out. Waiting for processes to exit.", diff --git a/filebeat/module/system/syslog/test/debian-12.journal-expected.json b/filebeat/module/system/syslog/test/debian-12.journal-expected.json index b75cce10fc8a..bdfbdb0d7975 100644 --- a/filebeat/module/system/syslog/test/debian-12.journal-expected.json +++ b/filebeat/module/system/syslog/test/debian-12.journal-expected.json @@ -8,6 +8,7 @@ "host.hostname": "vagrant-debian-12", "host.id": "5e6dc8fe417f4ea383e2afaa731f5d8a", "input.type": "journald", + "log.syslog.appname": "systemd", "log.syslog.facility.code": 3, "log.syslog.priority": 6, "message": "Stopped target getty.target - Login Prompts.", @@ -34,6 +35,7 @@ "host.hostname": "vagrant-debian-12", "host.id": "5e6dc8fe417f4ea383e2afaa731f5d8a", "input.type": "journald", + "log.syslog.appname": "kernel", "log.syslog.facility.code": 0, "log.syslog.priority": 6, "message": "Console: switching to colour frame buffer device 160x50", @@ -51,6 +53,7 @@ "host.hostname": "bookworm", "host.id": "5e6dc8fe417f4ea383e2afaa731f5d8a", "input.type": "journald", + "log.syslog.appname": "kernel", "log.syslog.facility.code": 0, "log.syslog.priority": 6, "message": "thermal_sys: Registered thermal governor 'power_allocator'", From 0e9fd0f6a33fda5a1c203666bd795129c7b3842c Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 31 Jan 2025 06:46:34 +0100 Subject: [PATCH 02/10] mergify: remove backport-8.x enforcement (#42510) --- .mergify.yml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/.mergify.yml b/.mergify.yml index aa850556a5d1..ea8948e485b0 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -147,20 +147,6 @@ pull_request_rules: branches, such as: * `backport-8./d` is the label to automatically backport to the `8./d` branch. `/d` is the digit - - name: add backport-8.x label for main only if no skipped or assigned already - conditions: - - -label~=^(backport-skip|backport-8.x)$ - - base=main - - -merged - - -closed - actions: - comment: - message: | - `backport-8.x` has been added to help with the transition to the new branch `8.x`. - If you don't need it please use `backport-skip` label and remove the `backport-8.x` label. - label: - add: - - backport-8.x - name: notify the backport has not been merged yet conditions: - -merged From 8f25b1dfaab026f2b9036c304c18853122f5b731 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Fri, 31 Jan 2025 08:37:30 +0100 Subject: [PATCH 03/10] Use centralized version qualifier (#42503) (#42506) * Use centralized version qualifier To avoid manual invocations specifying the version qualifier for prereleases in this commit we leverage a centralized version of truth for the version qualifier. * fix bug * revert defaults (cherry picked from commit ad052340f1f4aab382971eb27d8eade688444022) Co-authored-by: Dimitrios Liappis --- .buildkite/packaging.pipeline.yml | 26 ++++++++++++++++--------- .buildkite/scripts/version_qualifier.sh | 22 +++++++++++++++++++++ 2 files changed, 39 insertions(+), 9 deletions(-) create mode 100755 .buildkite/scripts/version_qualifier.sh diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index 2f3ff489e4ac..34b103b23736 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -61,7 +61,7 @@ steps: - label: Staging dashboards # TODO remove OR clause below (see earlier comment) - if: build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env('VERSION_QUALIFIER') != null + if: (build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env('VERSION_QUALIFIER') != null) || build.env('RUN_STAGING') == "true" depends_on: start-gate-staging key: dashboards-staging # TODO: container with go and make @@ -73,9 +73,10 @@ steps: retry: automatic: - limit: 1 - commands: - - make build/distributions/dependencies.csv - - make beats-dashboards + commands: | + source .buildkite/scripts/version_qualifier.sh + make build/distributions/dependencies.csv + make beats-dashboards env: SNAPSHOT: false DEV: false @@ -174,14 +175,16 @@ steps: key: packaging-staging depends_on: start-gate-staging # TODO remove OR clause below (see earlier comment) - if: build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env('VERSION_QUALIFIER') != null + if: (build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env('VERSION_QUALIFIER') != null) || build.env('RUN_STAGING') == "true" steps: - label: "STAGING: {{matrix}}" env: PLATFORMS: "${PLATFORMS}" SNAPSHOT: false DEV: false - command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" + command: | + source .buildkite/scripts/version_qualifier.sh + .buildkite/scripts/packaging/package-dra.sh {{matrix}} agents: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" @@ -214,7 +217,9 @@ steps: PACKAGES: "docker" SNAPSHOT: false DEV: false - command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" + command: | + source .buildkite/scripts/version_qualifier.sh + .buildkite/scripts/packaging/package-dra.sh {{matrix}} agents: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" @@ -244,7 +249,9 @@ steps: PLATFORMS: "${PLATFORMS}" SNAPSHOT: false DEV: false - command: ".buildkite/scripts/packaging/package-dra.sh x-pack/agentbeat" + command: | + source .buildkite/scripts/version_qualifier.sh + .buildkite/scripts/packaging/package-dra.sh x-pack/agentbeat agents: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" @@ -280,7 +287,7 @@ steps: - label: DRA Staging # TODO remove OR clause below (see earlier comment) - if: build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env('VERSION_QUALIFIER') != null + if: (build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env('VERSION_QUALIFIER') != null) || build.env('RUN_STAGING') == "true" key: dra-staging env: DRA_WORKFLOW: staging @@ -289,6 +296,7 @@ steps: - packaging-staging - dashboards-staging command: | + source .buildkite/scripts/version_qualifier.sh buildkite-agent artifact download "build/**" . .buildkite/scripts/packaging/prepare-release-manager.sh staging .buildkite/scripts/dra.sh diff --git a/.buildkite/scripts/version_qualifier.sh b/.buildkite/scripts/version_qualifier.sh new file mode 100755 index 000000000000..88d172dad588 --- /dev/null +++ b/.buildkite/scripts/version_qualifier.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +# An opinionated approach to managing the Elastic Qualifier for the DRA in a Google Bucket +# instead of using a Buildkite env variable. + +if [[ -n "$VERSION_QUALIFIER" ]]; then + echo "~~~ VERSION_QUALIFIER externally set to [$VERSION_QUALIFIER]" + return 0 +fi + +# DRA_BRANCH can be used for manually testing packaging with PRs +# e.g. define `DRA_BRANCH="main"` under Options/Environment Variables in the Buildkite UI after clicking new Build +BRANCH="${DRA_BRANCH:="${BUILDKITE_BRANCH:=""}"}" + +qualifier="" +URL="https://storage.googleapis.com/dra-qualifier/${BRANCH}" +if curl -sf -o /dev/null "$URL" ; then + qualifier=$(curl -s "$URL") +fi + +export VERSION_QUALIFIER="$qualifier" +echo "~~~ VERSION_QUALIFIER set to [$VERSION_QUALIFIER]" From 174c08dfe70b1fc3c1cc74d4f44ee1ebaa82716f Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Fri, 31 Jan 2025 09:44:16 +0200 Subject: [PATCH 04/10] Update branch filters to include/exclude 9.* (#42521) This commit updates branch_configuration and branch filters in CI pipeline definitions to support the new 9.* branches. --- catalog-info.yaml | 106 +++++++++++++++++++++++----------------------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/catalog-info.yaml b/catalog-info.yaml index 16d4bd7e0d0b..4ce530093d0b 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -39,7 +39,7 @@ spec: name: beats description: "Beats Main pipeline" spec: - branch_configuration: "main 7.17 8.*" + branch_configuration: "main 7.17 8.* 9.*" pipeline_file: ".buildkite/pipeline.yml" provider_settings: build_pull_request_forks: false @@ -51,9 +51,9 @@ spec: build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) repository: elastic/beats cancel_intermediate_builds: true - cancel_intermediate_builds_branch_filter: "!main !7.* !8.*" + cancel_intermediate_builds_branch_filter: "!main !7.* !8.* !9.*" skip_intermediate_builds: true - skip_intermediate_builds_branch_filter: "!main !7.* !8.*" + skip_intermediate_builds_branch_filter: "!main !7.* !8.* !9.*" env: # TODO set to true once https://github.com/elastic/ingest-dev/issues/3001 has been resolved ELASTIC_PR_COMMENTS_ENABLED: "false" @@ -89,7 +89,7 @@ spec: name: beats-metricbeat description: "Beats Metricbeat pipeline" spec: - branch_configuration: "main 7.17 8.*" + branch_configuration: "main 7.17 8.* 9.*" pipeline_file: ".buildkite/metricbeat/pipeline.yml" maximum_timeout_in_minutes: 120 provider_settings: @@ -102,9 +102,9 @@ spec: build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) repository: elastic/beats cancel_intermediate_builds: true - cancel_intermediate_builds_branch_filter: "!main !7.* !8.*" + cancel_intermediate_builds_branch_filter: "!main !7.* !8.* !9.*" skip_intermediate_builds: true - skip_intermediate_builds_branch_filter: "!main !7.* !8.*" + skip_intermediate_builds_branch_filter: "!main !7.* !8.* !9.*" env: # TODO set to truue once https://github.com/elastic/ingest-dev/issues/3001 has been resolved ELASTIC_PR_COMMENTS_ENABLED: "false" @@ -136,7 +136,7 @@ spec: name: filebeat description: "Filebeat pipeline" spec: - branch_configuration: "main 7.17 8.*" + branch_configuration: "main 7.17 8.* 9.*" pipeline_file: ".buildkite/filebeat/filebeat-pipeline.yml" maximum_timeout_in_minutes: 120 provider_settings: @@ -149,9 +149,9 @@ spec: build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) repository: elastic/beats cancel_intermediate_builds: true - cancel_intermediate_builds_branch_filter: "!main !7.* !8.*" + cancel_intermediate_builds_branch_filter: "!main !7.* !8.* !9.*" skip_intermediate_builds: true - skip_intermediate_builds_branch_filter: "!main !7.* !8.*" + skip_intermediate_builds_branch_filter: "!main !7.* !8.* !9.*" env: # TODO set to truue once https://github.com/elastic/ingest-dev/issues/3001 has been resolved ELASTIC_PR_COMMENTS_ENABLED: "false" @@ -196,9 +196,9 @@ spec: build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) repository: elastic/beats cancel_intermediate_builds: true - cancel_intermediate_builds_branch_filter: "!main !7.* !8.*" + cancel_intermediate_builds_branch_filter: "!main !7.* !8.* !9.*" skip_intermediate_builds: true - skip_intermediate_builds_branch_filter: "!main !7.* !8.*" + skip_intermediate_builds_branch_filter: "!main !7.* !8.* !9.*" env: # TODO set to true once https://github.com/elastic/ingest-dev/issues/3001 has been resolved ELASTIC_PR_COMMENTS_ENABLED: "false" @@ -230,7 +230,7 @@ spec: name: heartbeat description: "Heartbeat pipeline" spec: - branch_configuration: "main 7.17 8.*" + branch_configuration: "main 7.17 8.* 9.*" pipeline_file: ".buildkite/heartbeat/heartbeat-pipeline.yml" maximum_timeout_in_minutes: 120 provider_settings: @@ -243,9 +243,9 @@ spec: build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) repository: elastic/beats cancel_intermediate_builds: true - cancel_intermediate_builds_branch_filter: "!main !7.* !8.*" + cancel_intermediate_builds_branch_filter: "!main !7.* !8.* !9.*" skip_intermediate_builds: true - skip_intermediate_builds_branch_filter: "!main !7.* !8.*" + skip_intermediate_builds_branch_filter: "!main !7.* !8.* !9.*" env: # TODO set to true once https://github.com/elastic/ingest-dev/issues/3001 has been resolved ELASTIC_PR_COMMENTS_ENABLED: "false" @@ -290,9 +290,9 @@ spec: build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) repository: elastic/beats cancel_intermediate_builds: true - cancel_intermediate_builds_branch_filter: "!main !7.* !8.*" + cancel_intermediate_builds_branch_filter: "!main !7.* !8.* !9.*" skip_intermediate_builds: true - skip_intermediate_builds_branch_filter: "!main !7.* !8.*" + skip_intermediate_builds_branch_filter: "!main !7.* !8.* !9.*" env: # TODO set to true once https://github.com/elastic/ingest-dev/issues/3001 has been resolved ELASTIC_PR_COMMENTS_ENABLED: "false" @@ -324,7 +324,7 @@ spec: name: beats-libbeat description: "Beats libbeat pipeline" spec: - branch_configuration: "main 7.17 8.*" + branch_configuration: "main 7.17 8.* 9.*" pipeline_file: ".buildkite/libbeat/pipeline.libbeat.yml" maximum_timeout_in_minutes: 120 provider_settings: @@ -337,9 +337,9 @@ spec: build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) repository: elastic/beats cancel_intermediate_builds: true - cancel_intermediate_builds_branch_filter: "!main !7.17 !8.*" + cancel_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" skip_intermediate_builds: true - skip_intermediate_builds_branch_filter: "!main !7.17 !8.*" + skip_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" env: # TODO set to true once https://github.com/elastic/ingest-dev/issues/3001 has been resolved ELASTIC_PR_COMMENTS_ENABLED: "false" @@ -371,7 +371,7 @@ spec: name: beats-packetbeat description: "Beats packetbeat pipeline" spec: - branch_configuration: "main 7.17 8.*" + branch_configuration: "main 7.17 8.* 9.*" pipeline_file: ".buildkite/packetbeat/pipeline.packetbeat.yml" maximum_timeout_in_minutes: 120 provider_settings: @@ -384,9 +384,9 @@ spec: build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) repository: elastic/beats cancel_intermediate_builds: true - cancel_intermediate_builds_branch_filter: "!main !7.17 !8.*" + cancel_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" skip_intermediate_builds: true - skip_intermediate_builds_branch_filter: "!main !7.17 !8.*" + skip_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" env: # TODO set to true once https://github.com/elastic/ingest-dev/issues/3001 has been resolved ELASTIC_PR_COMMENTS_ENABLED: "false" @@ -430,9 +430,9 @@ spec: build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) repository: elastic/beats cancel_intermediate_builds: true - cancel_intermediate_builds_branch_filter: "!main !7.17 !8.*" + cancel_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" skip_intermediate_builds: true - skip_intermediate_builds_branch_filter: "!main !7.17 !8.*" + skip_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" env: # TODO set to true once https://github.com/elastic/ingest-dev/issues/3001 has been resolved ELASTIC_PR_COMMENTS_ENABLED: "false" @@ -464,7 +464,7 @@ spec: name: beats-winlogbeat description: "Beats winlogbeat pipeline" spec: - branch_configuration: "main 7.17 8.*" + branch_configuration: "main 7.17 8.* 9.*" pipeline_file: ".buildkite/winlogbeat/pipeline.winlogbeat.yml" maximum_timeout_in_minutes: 120 provider_settings: @@ -477,9 +477,9 @@ spec: build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) repository: elastic/beats cancel_intermediate_builds: true - cancel_intermediate_builds_branch_filter: "!main !7.17 !8.*" + cancel_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" skip_intermediate_builds: true - skip_intermediate_builds_branch_filter: "!main !7.17 !8.*" + skip_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" env: # TODO set to true once https://github.com/elastic/ingest-dev/issues/3001 has been resolved ELASTIC_PR_COMMENTS_ENABLED: "false" @@ -511,7 +511,7 @@ spec: name: beats-xpack-winlogbeat description: "Beats x-pack winlogbeat pipeline" spec: - branch_configuration: "main 7.17 8.*" + branch_configuration: "main 7.17 8.* 9.*" pipeline_file: ".buildkite/x-pack/pipeline.xpack.winlogbeat.yml" maximum_timeout_in_minutes: 120 provider_settings: @@ -524,9 +524,9 @@ spec: build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) repository: elastic/beats cancel_intermediate_builds: true - cancel_intermediate_builds_branch_filter: "!main !7.17 !8.*" + cancel_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" skip_intermediate_builds: true - skip_intermediate_builds_branch_filter: "!main !7.17 !8.*" + skip_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" env: # TODO set to true once https://github.com/elastic/ingest-dev/issues/3001 has been resolved ELASTIC_PR_COMMENTS_ENABLED: "false" @@ -558,7 +558,7 @@ spec: name: beats-xpack-packetbeat description: "Beats x-pack packetbeat pipeline" spec: - branch_configuration: "main 7.17 8.*" + branch_configuration: "main 7.17 8.* 9.*" pipeline_file: ".buildkite/x-pack/pipeline.xpack.packetbeat.yml" maximum_timeout_in_minutes: 120 provider_settings: @@ -571,9 +571,9 @@ spec: build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) repository: elastic/beats cancel_intermediate_builds: true - cancel_intermediate_builds_branch_filter: "!main !7.17 !8.*" + cancel_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" skip_intermediate_builds: true - skip_intermediate_builds_branch_filter: "!main !7.17 !8.*" + skip_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" env: # TODO set to true once https://github.com/elastic/ingest-dev/issues/3001 has been resolved ELASTIC_PR_COMMENTS_ENABLED: "false" @@ -605,7 +605,7 @@ spec: name: beats-xpack-libbeat description: "Beats x-pack libbeat pipeline" spec: - branch_configuration: "main 7.17 8.*" + branch_configuration: "main 7.17 8.* 9.*" pipeline_file: ".buildkite/x-pack/pipeline.xpack.libbeat.yml" maximum_timeout_in_minutes: 120 provider_settings: @@ -618,9 +618,9 @@ spec: build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) repository: elastic/beats cancel_intermediate_builds: true - cancel_intermediate_builds_branch_filter: "!main !7.17 !8.*" + cancel_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" skip_intermediate_builds: true - skip_intermediate_builds_branch_filter: "!main !7.17 !8.*" + skip_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" env: # TODO set to true once https://github.com/elastic/ingest-dev/issues/3001 has been resolved ELASTIC_PR_COMMENTS_ENABLED: "false" @@ -652,7 +652,7 @@ spec: name: beats-xpack-metricbeat description: "Beats x-pack metricbeat pipeline" spec: - branch_configuration: "main 7.17 8.*" + branch_configuration: "main 7.17 8.* 9.*" pipeline_file: ".buildkite/x-pack/pipeline.xpack.metricbeat.yml" maximum_timeout_in_minutes: 120 provider_settings: @@ -665,9 +665,9 @@ spec: build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) repository: elastic/beats cancel_intermediate_builds: true - cancel_intermediate_builds_branch_filter: "!main !7.17 !8.*" + cancel_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" skip_intermediate_builds: true - skip_intermediate_builds_branch_filter: "!main !7.17 !8.*" + skip_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" env: # TODO set to true once https://github.com/elastic/ingest-dev/issues/3001 has been resolved ELASTIC_PR_COMMENTS_ENABLED: "false" @@ -782,9 +782,9 @@ spec: build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) repository: elastic/beats cancel_intermediate_builds: true - cancel_intermediate_builds_branch_filter: "!main !7.17 !8.*" + cancel_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" skip_intermediate_builds: true - skip_intermediate_builds_branch_filter: "!main !7.17 !8.*" + skip_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" env: # TODO set to true once https://github.com/elastic/ingest-dev/issues/3001 has been resolved ELASTIC_PR_COMMENTS_ENABLED: "false" @@ -829,9 +829,9 @@ spec: build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) repository: elastic/beats cancel_intermediate_builds: true - cancel_intermediate_builds_branch_filter: "!main !7.17 !8.*" + cancel_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" skip_intermediate_builds: true - skip_intermediate_builds_branch_filter: "!main !7.17 !8.*" + skip_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" env: # TODO set to true once https://github.com/elastic/ingest-dev/issues/3001 has been resolved ELASTIC_PR_COMMENTS_ENABLED: "false" @@ -876,9 +876,9 @@ spec: build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) repository: elastic/beats cancel_intermediate_builds: true - cancel_intermediate_builds_branch_filter: "!main !7.17 !8.*" + cancel_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" skip_intermediate_builds: true - skip_intermediate_builds_branch_filter: "!main !7.17 !8.*" + skip_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" env: # TODO set to true once https://github.com/elastic/ingest-dev/issues/3001 has been resolved ELASTIC_PR_COMMENTS_ENABLED: "false" @@ -923,9 +923,9 @@ spec: build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) repository: elastic/beats cancel_intermediate_builds: true - cancel_intermediate_builds_branch_filter: "!main !7.17 !8.*" + cancel_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" skip_intermediate_builds: true - skip_intermediate_builds_branch_filter: "!main !7.17 !8.*" + skip_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" env: # TODO set to true once https://github.com/elastic/ingest-dev/issues/3001 has been resolved ELASTIC_PR_COMMENTS_ENABLED: "false" @@ -970,9 +970,9 @@ spec: build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) repository: elastic/beats cancel_intermediate_builds: true - cancel_intermediate_builds_branch_filter: "!main !7.17 !8.*" + cancel_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" skip_intermediate_builds: true - skip_intermediate_builds_branch_filter: "!main !7.17 !8.*" + skip_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" env: # TODO set to true once https://github.com/elastic/ingest-dev/issues/3001 has been resolved ELASTIC_PR_COMMENTS_ENABLED: "false" @@ -1017,9 +1017,9 @@ spec: build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) repository: elastic/beats cancel_intermediate_builds: true - cancel_intermediate_builds_branch_filter: "!main !8.*" + cancel_intermediate_builds_branch_filter: "!main !8.* !9.*" skip_intermediate_builds: true - skip_intermediate_builds_branch_filter: "!main !8.*" + skip_intermediate_builds_branch_filter: "!main !8.* !9.*" env: # TODO set to true once https://github.com/elastic/ingest-dev/issues/3001 has been resolved ELASTIC_PR_COMMENTS_ENABLED: "false" @@ -1053,7 +1053,7 @@ spec: spec: repository: elastic/beats pipeline_file: ".buildkite/packaging.pipeline.yml" - branch_configuration: "main 8.* 7.17" + branch_configuration: "main 8.* 7.17 9.*" cancel_intermediate_builds: false skip_intermediate_builds: false maximum_timeout_in_minutes: 90 @@ -1102,7 +1102,7 @@ spec: spec: repository: elastic/beats pipeline_file: ".buildkite/ironbank-validation.yml" - branch_configuration: "main 8.* 7.17" + branch_configuration: "main 8.* 7.17 9.*" cancel_intermediate_builds: false skip_intermediate_builds: false provider_settings: From 0ff52ebd4dd6fee948f790091248ae9cfbef0cb8 Mon Sep 17 00:00:00 2001 From: William Easton Date: Fri, 31 Jan 2025 01:59:25 -0600 Subject: [PATCH 05/10] Write latency statistics go under a histogram key (#42514) * Write latency goes under histogram * Update fields.yml to include new exposed beats stats * Add't updates for integ tests --------- Co-authored-by: Julien Lind --- metricbeat/docs/fields.asciidoc | 134 ++++++++++++++++++ metricbeat/module/beat/fields.go | 2 +- metricbeat/module/beat/stats/_meta/fields.yml | 78 +++++++++- metricbeat/module/beat/stats/data.go | 11 +- 4 files changed, 216 insertions(+), 9 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 58ee16f2c838..3d571d94d63a 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -10184,16 +10184,113 @@ type: long -- + *`beat.stats.libbeat.pipeline.queue.acked`*:: + -- +Number of acknowledged events + + +type: long + +-- + +*`beat.stats.libbeat.pipeline.queue.added.bytes`*:: ++ +-- +Number of bytes added to the queue + + +type: long + +-- + +*`beat.stats.libbeat.pipeline.queue.added.events`*:: ++ +-- +Number of events added to the queue + + +type: long + +-- + +*`beat.stats.libbeat.pipeline.queue.consumed.bytes`*:: ++ +-- +Number of bytes consumed from the queue + + type: long -- +*`beat.stats.libbeat.pipeline.queue.consumed.events`*:: ++ +-- +Number of events consumed from the queue + + +type: long + +-- + +*`beat.stats.libbeat.pipeline.queue.filled.bytes`*:: ++ +-- +Number of bytes filled in the queue + + +type: long + +-- + +*`beat.stats.libbeat.pipeline.queue.filled.events`*:: ++ +-- +Number of events filled in the queue + + +type: long + +-- + +*`beat.stats.libbeat.pipeline.queue.filled.pct`*:: ++ +-- +Percentage of the queue filled + + +type: float + +-- + *`beat.stats.libbeat.pipeline.queue.max_events`*:: + -- +Maximum number of events allowed in the queue + + +type: long + +-- + +*`beat.stats.libbeat.pipeline.queue.removed.bytes`*:: ++ +-- +Number of bytes removed from the queue + + +type: long + +-- + +*`beat.stats.libbeat.pipeline.queue.removed.events`*:: ++ +-- +Number of events removed from the queue + + type: long -- @@ -10437,6 +10534,43 @@ type: long +*`beat.stats.libbeat.output.write.latency.histogram.count`*:: ++ +-- +type: long + +-- + +*`beat.stats.libbeat.output.write.latency.histogram.max`*:: ++ +-- +type: float + +-- + +*`beat.stats.libbeat.output.write.latency.histogram.median`*:: ++ +-- +type: long + +-- + +*`beat.stats.libbeat.output.write.latency.histogram.p95`*:: ++ +-- +type: float + +-- + +*`beat.stats.libbeat.output.write.latency.histogram.p99`*:: ++ +-- +type: float + +-- + + + *`beat.stats.output.elasticsearch.bulk_requests.available`*:: + diff --git a/metricbeat/module/beat/fields.go b/metricbeat/module/beat/fields.go index f0a40fd701af..45e1586c4e26 100644 --- a/metricbeat/module/beat/fields.go +++ b/metricbeat/module/beat/fields.go @@ -32,5 +32,5 @@ func init() { // AssetBeat returns asset data. // This is the base64 encoded zlib format compressed contents of module/beat. func AssetBeat() string { - return "eJzsXUtv5LgRvs+vIHzOCpjDXHzYBIskwF4SIFgghyAw2Orqbq4pUkNSnnV+fSCpW60HHyWJktUz8mUX9uirr0pSPSiy6ifyCu/P5ADUfCLEMMPhmTz9AtQ8fSLkCDpVLDdMimfy8ydCCCn/RDJ5LDh8IkQBB6rhmZzpJ0I0GMPEWT+T/zxpzZ/+RJ4uxuRP/y3/dpHKvKRSnNj5mZwo1+X1Jwb8qJ8r5J+IoBnUXPSLNtTo6veEmPe8FKFkkV9/076ufS3Ns580qDdQzZ9sl9sgOjBp1vm9C8OF08ZS8LUAbZJUFsIM/tUNmXJGteWvOTWX2iRJZZGE5tlLrWFC0yzxo9856FyKyuJ28TbFfMq1wV2KhZXDKVhTd9rvTgSUksolx69nSNeOvlxqODr/GUbpUYrXaiUBsZi7sSA5r9QbtyOk8mh7CJclFxB7Y3eS6sCORxCrEwxLvnFkwoASlK9OMSj4zvCNcnb8WoB6/wCWCOE3phmYi1z/PQ6IvbET0pxkIdbnFxR8Y/i1gGL9l9kvtQl41ABnGVvfE4Yl3zgaloEs1mcYktvwk5JTdV7/FgcF3xgWghbmIhX73wfEY5TwFtM3yjg98PXNiZF941m5T2rWJ+kV3GG3eH5H0xRys+YTVemVBOV+UIpXs8NleEKaTB7Zia1vPozoG035ujY7i8S7b9AQvzAcojbP9xmESU/naBUucKoNSzVQlV6WKTFpeoEEhFEM/MXGzILzapqko1KCFd+lq+CkQF+SE2W8UOAvTBdhjGJgJ62LNAWtP5K1n0JTtYFJL8m6PL0yu8ROlPMDTV3+ZkF6XsldkteyaX2OPsFdiuEkZkGaLuE3ir9TOHdWHMksf3pWebrQSh3nkPpdZ4wMKrTk2ZU4J9LWlk9KiyVX7QJLokOuzqXRLk2fYQjCOGREBkfQxiGTDBRcUu0T9i6tdhmH7ESQtiIj7UVG2YzMsRtq0bOvA2ZJ4mP0wDPDL118jCZYXiMXED5GGTS5TnmMeipX8C0hMjfSmmY5Z6JfjXSZxghK8Aai5JcCe4OwmeLGppuWySgWHxtPG857QA1baGJE/aC3dUgcTWcLzmZI38WmWfAxvP+obTA/5/K8/Ofr9ZxJafT6hpWK7W7Ea5s9JScTTfZA2XhAhcdIxANKPEwOHtBj8+m3iz82887AKJausF9KCl1koJJC6CLPpTJwfDkqmecLfyS6G+iqajKJykcGzhvxPXYGzLOHTzLdag8ZQV1aPFoQdenxgHHUpcoDhdKBCthoahRNA18hH7V6q1XbY1DAOnsIIpON9pARyKHEowUghxoPGH8cmjxQ+Olr4GNyY3sxxuYa9uVIm5VLY+3LkWHb7MGMTDTZo4UytwoPFMjcSjxWGHPr8RhBzMJ/X44cGOgRlyM7xPfYGTDPHj7JdKs9ZAT9DpYjvXo8YBx97OVIuwo/9nJkZZN9ORJlnT0EkclGe8gI9PjLkT41HjD+PPRypFWD0DbLXMkUtJbxzkJVNit5CH2SKqOGSWHTbfph2YZzgpPVrVdXooYU1myqz6lYiRlKVMPLKKD91lokymJ14PT8vNOBLXUrDfxn9VfuR+U/vYlTfrQBug13ghxX6SXiIukU3kqWhaZp+eyu9NZgJTa5hZR93z/dpy7Xka6k+X23pLtq+AP2pOtqvrWmdHZ2W+lKZ2W3qbZ0VoYb60tn5bi9xnQOmtvqTGcluaXWdFaCG+pNZ+W3peZ0VoLb6k5npbip9nR2hpvrT2eluckGdS6m2+pQZ2X5o7Wo6xphcz3qbPS21aTOxnBDXeps9FZtU1cRcPepa055t4pl7dge4SbhI4CX0HDq95YnG63Ebyb+nmvxRscfsBrv6761etzFbysVuYPfpmpyB8eNVeUOltury51Et1WZO2huqTZ3UNxQde5guKX63EFxWxW6g+SmanQXx81V6Q6im6zT3Vy3Vak7eP5otXrfDJur1u0Et1Wv2zluqGK3E1y1Zr/+pwt8E3eop8h1pYyZu3aR2v6ZG1XZM3GSyQCieYpo1ncHY7EHEE2ce8/nYg8gmntYDJzUWOwBROMBQWkm+4XEWPg+ivtEz/gnQr9rA/GG8aV5kaTS3gp/5EtRM0vciPdjtPSYfI4m0AHXkfYlrrQhXFe3yOIseB15Qqpl9pA9fX6Kt1DV1qhkbH0E2rK/LCrcdo9a0hcV/sX6anN26AUMMuvdzlkOnImFVjE5A2H8S4ejjHTVPrmxTnwSOoVzQtPXmPscB0xCUprV1DcIrOzGyZ8Ne1ugiBioXamjk4C8Zo1yqROjLl7Yc6EnyviavALyGlqMG1CrEgtJbBxHceBMX9bkFhZ5/zZjllgXdRHzi7uv65glVpVdpOzimqKrMLl1LWq+16/lr+Di3A6dzDdnbaC7f/MJW8vr9jmhfO6BmvTiPNUYnVVI3GqhoEsLGwiORc5ZShdZenIwu0kM22zpKNUlhoxRRsqMigW9bZdVSNxK3rZPyiXsHpHoMTm8u+/yjBz0yiUgosME8U1/NhWPDM8SUDwSTvCb9G+KGVj4poRkdLksfFu8QlqtXk7MNllmfmaQyWPBIVGFEKHhNZPUrKl78XtUtKFqiQL1ysQDPyAi8yV59NEdq89k1mJCtbAJ+QUyUJS/WL8TjVziCkN2hBe5YRkkWYSlygBes/pOxZF7lkZn9RJkGTPJhaqICxdXvkkAu0tBy1PE3WhdCk7splTJnbt2Zgi3ot4X4bPqXy9yV8/pi4A/ItrzxjbxIbdUk+r9hXIubfN45lKo4RM3fI+HL02LwAORmOmIbrcRb0NtAqzr4Ymw4JoXi9fdSxyark1SfRZCnItOTzrJQTF5TIoF6toWGZykNq+vhTR0DVpBQff2AXZH1qUTs4NPbTGfTII2AwmYov4FRmJTc1yUNIbDEXVVXJ7jZA8ZW4/WL0/WIbbldWiaGverO8GT3tn4oPsMKnfvttJ8Hl4B3dC2hDn8yC3519wqcm3bZYER06ZUaHqOXm4PKfnE+OPk/Ph7/YhrWPoaU8fylQwhDyi4yqAYJNzYneW35I1y59bhiRxCwF0CC9yJEHCfwAL3IQzd7IPSoJYwQgC3J34BE3SQP/XlDr5ujd9DBZxqw1INVKWXaCskh4K/vlzPly1TXYa2Gk+w922tu22RpKNJ4pd6X1rMcg5R+00huLml3peOjvAHWBdeI9wQ32e6WBrfNPB9FGzuggLP963ojHzi7mfXtFHyfT1SXYHtxVD9Ul5/s+Dw9vdv+Sa28UJS/e8i23iv2Its471iL7KN94q91DbeK7xrG28ZmrShWT54JvrYNe7TX5orngZg7SeSTHq47Lcw4lPgfHqnRMnyXWUGUlMo98ZE/0J+ySfx4rSZWywzSpATw/MCjhLgBZfLBK6cU3Oy7xoOcye2R6hSRerEi+x/PyNItwHfM4EyaNhO2Ix4U1wgXSHl//ehRgsZgvS+K85UxYnSEzNTGSdKN6Gf6cFcII2bBfXGUruASZ9DZ3//hOTKybbkE8G3WAR5PY37nZworQ1oyaSGtj+CThXLDZPimfwcyMk6RquxXuH9m+x887Qi1j+/ADXk178mw+jeTVimQpfXDcG7KWvKC21AJQhdxqULIXJ9FAUcqC6BKDLpWPF9Gtq+e6n3HQldHHrqbdd3fH00C/hb3nBp2fTStoA7T/CpEHWLiCVP9LPo6zD5Ysd2MNzFlkIEf/Gc5+f2+jsrId/FUZPyVApDmQDXkXucLRbKVV9BCfDvI3AxI+EHbBwIKm/GAGEyYN/tz6igZ8hAmAQEPQw389YQByk50L4YT3gof37VJAVhFOUtMeQq5s/TMlImDJwHjcICRP5RZAdQRJ6u+JrYdEWnqz57hpJQ+z0N8acZlOxrcFJoOJLDexWCrSTqc2bLcKiwyQGYONuJuPYUTM8w9KwM437GPpqbo+kyx2Qxk5c8YbwL5R3LFG+rD67DhZNvGyo4Jyn+NpRgQzmkCmSERSZABvu2TcDEtFqbAItojTYJFdXLbAJysPHYBExEr7AJqKHmXhMgsSOsRsLiRkqNBQ23vJqAOmKa1XhkREOpCcCI9k9I1A7iqn4X0bsJqQNZ1vPiWiRNAPZ2NELg3R8zeyeiAERzI84gTHpa5iyPb38ETkpIUltaStMLJGXBwSA8YnDE61EDKzgp0JfqyGNhb3QzG1sXaQpaxwI/gUkvSVy0E+X8QNNwP64RmNhBWyMgx7Tz874gv1NLvUmivB5nlfuPV8TJzzmHNPwmxPTu2OG1ZELGEh43i9eJLD5NNagXGVP/dLEx2pERGpIJWpJxmpIJuedMMfiZqzOEjJyJOlJSJ01Dzs4d5TJv7fpXdRLVsfhEQQrsDcZMYR3hK3Y3NNkNxXzO+hKiPchNMm34MoflVgnRXJ7XXfXaX4w9PgdF7PF5M/F52O/VDhZ1ZVwKXWR1R+Iiz6UycHTMCJqgGdm9kYXB7o2cInZvtBlvZBRNEWtKe8Li/9ldhJ/N7iIey0U0G8qMcT1Ee8lkh9490O6BRojZPdBeMu3eaPdGARG7N9qMN9pLJu/P7iJsVHYX8T27iOYkiJIpaC2X2WJSPc71FPaTVBmtBrF7q7NgfVdnVnExdU5FZESjgPqP16w6g3KrG+DDW6+QOpDoW3VbwVNompYPxfxnpPF8Ui4z2WQ/w7Gf4bBC7mc49jMc+xmO/QzHPY3cz3BYUPczHGiI/vbNpJUpac/CIwbSdmCW7AmS42dPkCJA7gnSniDtCdKeIO0Jkhd1T5BQEHenepKfbFdMafwh5jSXmtXWyjxkZ6qobcF24+Oub3L4ahJDNPNXQ4Wk/eAzoqLgkrrNMKeeePr85PUMR1mEes4/ff4yH2M+hAg1BItRyrjNheJKcGYbjzUNqvVwRnvQA2NUMNVzaAYKAsM/wwQNMEOL0AQRBIR3BAj++pEMFgn9qJGlwVgQHD3qVMY6EXCWN1/EG2Om4SGTPsQEOyRSYMoWps+ib0JePOccnvg2ZtPRiEFyk2C9k+YQb7d3cBvuhULMXEMQqSeDLfI+RHj0xoxqQ+zewA9Zc/oix+jb6d7IPZQWffec42TxCK5BsJjtBNYJsQEz2sdCT7eiYwgyJjn3DXpGX++Y0hywgismOsUGGpVWbUNrUKs8VYhK4FkqWRgmHDdgvOB7l9k7NLnOmSdMuNu1Xgexo5+DAI+/V48JSWWWSUGMJJTzSnhf0dDzlLMcOBPu/HNWosAZiEBIDXqzutEtTV8DU6GQQBn946VqSzKTFgIjzuYzz9QwBN02VviYwgiwE2XDRtJTsRg3oGKh5cWBM32JBafAeD/VjEmrPEPOA1D3lWFxYjZJET7a1m5s3muhDVVzXy1tZD4TQgGX1JkNI+w8mBzZvdZl54DbLn/+WTf4dhUbmDvlWKskqJQTRbL8+e09b3UkX9IZIgn9rZRSfxuxD2kkoxyrO6AQ7FcQJHHSyRxqY5UEhPzG4Xj28IgZBibSrUX7b3HrawA16WVuN8o5hg0yiBoN5z0CIQoN1SLnLKXuWm4dtjcWYcIxkoN5ZAMMWt/iMypmhveJTI2UpBSOfbVmZA+TmP5WyiOiZ9lA2LV+ACIxo8C/gB69S3XYCOBbHCGL3vlKss9Yo7eSLcGy5OeTf6P4TTHnHo5od/3fpZDv4raX5jKezWobuPPVHbUR8GbHM767BHqIz69tQvPWw1JCktrSZral7sOFxqUj4O6fy5wTzslm1y38g8pHgoVGjPfg/h8AAP//94XAGA==" + return "eJzsXUuPpLiW3tevsHI9jVSLWnQuekatmZF6MQ+NWprF1VXKASci3GkwZZusyvvrr4CA4OHHAQxJVJGbblUm3/nOAc4L+/gX8grvz+QEVH8iRDPN4Zk8/Q5UP30iJAEVS5ZrJrJn8tsnQggpf0VSkRQcPhEigQNV8Ewu9BMhCrRm2UU9k789KcWf/oU8XbXOn/5e/u4qpH6JRXZml2dyplyV158Z8EQ9V8i/kIymUHNRL0pTrap/J0S/56UIKYr89i/d67rX0jz9RYF8A9n+ynS5CaIHE6e9f7dh2HC6WBK+FqB0FIsi06O/apApZ1QZfptTfa1NElUWiWievtQaRjROIzf6nYPKRVZZ3CzepJhLuS64TTG/cjgFa+pW+92JgJRC2uS49fTp2tOXCwWJ9c8wSk9SvFYr8ojF3I0VyTmlNtwSiEViegjXJecR27A7C3liSQLZ5gT9khuOLNMgM8o3p+gVfGf4RjlLvhYg3z+AJUJ4wzQFfRXbv8cesQ27TOizKLLt+XkFNwy/FlBs/zK7pbYBj2rgLGXbe0K/5IajZimIYnuGPrktPyE4lZftb7FXcMOwyGihr0Kyf3xAPEYJ7zB9o4zTE9/enBjZDc/KfVK9PUmn4B671fM7GseQ6y2fqEqvyCv3g1K8mh0uw8uETkXCzmx782FENzTF69bsDBLvvkFB+MJwjNo+3xfIdHy+BKtwgVOlWayAyvi6TolJ4ytEkGnJwF1sLCw4b6aJeipFWPF9uhLOEtQ1OlPGCwnuwnQVxigGZtKqiGNQ6iNZuym0VRvo+Bpty9Mps0/sTDk/0djmb1ak55TcJ3krm7bn6BLcp+hPYlakaRPeUPyLwqXXcSSL/OlF5vFKnTrOIXa7zhAZlK/l2Ze4JNLWlo9Ki0U37Twt0TFXa2u0T9NlGIIwDpmQwRG0ccgsA3lbqkPCztZqn7HPTgRpKzLRXmSSzcgSu6GankMdMC2Jj9EDzwzfuvgYTbC8JjYQPkYZNLleeYx6KjfwLT4yDWlF05yzbFiN9JmGCErwBlnJLwb2Bn4zhY1NjZbRJBYfG09bzkdA9VtoZkT9oLd1TBxNZw/OZkzfxqZt+Gg+fNR2mJ9zcVn/8/V2zqQ0en3DSsUON+K0zZGSk5kme6Bs3KPCYyTiHiUeJgf36LH79NvGH5t5p6AlizdYLyUyVaQgoyJTRZ4LqSF5SaTI85U/Et0NdFM1mkXlIwNnQ/yInR7zHOGTzLfaQ0ZQmxaPFkRtejxgHLWp8kChdKQCNppqSWPPV8hHrd5q1Y4Y5LHOEYLIbKM9ZASyKPFoAciixgPGH4smDxR+hhq4mDRsr1qbXMPRjjRZuTTW0Y702+YIZmSmyR4tlNlVeKBAZlfiscKYXY/HCGIG/kc7cmSgR2xH9ogfsdNjniN8kvlWe8gI+gO0I516PGAcfex2pFmFn7sdWdnkaEeirHOEIDLbaA8ZgR6/HelS4wHjz0O3I40a+JZZ5lLEoJQItxeqslnJI1NnIVOqmchMus3fLNtyjnCy+vXqRtSQwtpF9TnNNmKGEtXy0hLocLQWCdKs9uyeX7Y7sKNupYF7r/7G86jcuzdxyk82QH/gjpfjJrNEbCStwjvJcqZoXD67G701WIltbiHE0PfP96nrTaQraf7YI+luGv6EM+n6mu9tKJ2Z3V6m0hnZ7WosnZHhzubSGTnubzCdhea+JtMZSe5pNJ2R4I5m0xn57Wk4nZHgvqbTGSnuajydmeHu5tMZae5yQJ2N6b4m1BlZ/mwj6vpG2N2MOhO9fQ2pMzHc0ZQ6E71Nx9RVBOxz6tpd3p1iWVmWR9hJuAjgJbSchrPlyU4r8cbEP3It3ur4E1bjQ933Vo/b+O2lIrfw21VNbuG4s6rcwnJ/dbmV6L4qcwvNPdXmFoo7qs4tDPdUn1so7qtCt5DcVY1u47i7Kt1CdJd1up3rvip1C8+frVYfmmF31bqZ4L7qdTPHHVXsZoKb1uy3//SBG3Gn+hS5vpQp565dhTJ/5kZV9iw7i2gE0T5FNB26g6nYI4g2zr3nS7FHEO09LEZOair2CKL1gCAVE8NCYir8EMW+o2f6E6HelYZwh/HFeRHFwjwKf+JLUTOL7Ij3bbQ0iT4HE2iB60n7ElbaGK6vW2BxBryevEzIddaQPX1+Cteo6mpUMjY+Al3ZX1YVbrpHHemrCv9ifLU5Ow0CBln0bucsB86ylbqYnEGm3a3DSUa6aR81rCOXhF7hHNH4NeQ6xxETn5S2m/oGns5umPxZs7cVioiR2pU6KvLIa3uUa+0YtfHC7gs9U8a35OWR19JiXIPclJhPYus4ihNn6rolN7/I+7cZvUZf1EbMLe7e19FrdJVtpMzi2qKr0LmxF7Xc69fyN3BxdodOlpuzNtDdv7mEbeV1h5xQPvdEdXy17moMzsonbrNQ0KeFDQRJkXMW01VaTxZmjUS/zdaOUn1iyBilhUhptqK37bPyidvI2w5J2YTdIxJNotO7/S4vyEFvXDwiekwQ3/QXU3HIcLSAwpGwgjfSv0mmYeWb4pPR57LybXEK6Yx6OTPTyTLLM4NUJAWHSBZZ5ju8ZpaaNXUn/oCK0lSuUaDemDjgR0REviaPIbql+0wWNROqxibkV0hBUv5i/E40scXlh+wJL3LNUojSAK1KD17bfadZwh2t0UWzBFnKdHSlMmDj4sY38mD3KShxDrgarU/Bit2WKrl11c4C4UbUexM+rf56lbt6iV8y+B7Qng3byIXcUU3I9xfKuTCdx7OUQg0f2eEHPFxpWgAeiMRMBXS7rXgTahtgbQ9PgIZrXqxed6+xabo2SfVZCLEvOj6rKAfJRBIVK9S1HTI4SV1eXwuh6Ra0vILu4wPMjqxPJ+QEn9piLpkEbQbiMUX9DxiJbc1xlUJrDgnqqrA8p8keMzZurV+frEVsx+vQONb2V3eGJ72zcUEPGVTu3m6l5TycAvqhbQ1zuJE78m+5VeDats8CI6ZLqVD0ErzcHlNyiXHHyeXx9/YRV7P4NaSO5SvpQx5RsJVBIUjYsXvtt+iNcuvS4ZkcfMB9AivcCR/wkMAK98EP3a6DUiDXMIIHdyB+BRP0kD8N5Y6+bk1fQwWcKs1iBVTG12AdklPBX19u+8vWqS59S41n2LvpdXctEvU0idxS763FNOcQdN4Ugptd6r11lMB3MDZeA9wQ12e6UBo3Grg+CrZ3QYLj+1ZwRi5x971rSkvxvh2pvsBuM1S9lNc3Fhzf/uEt38UyXoiq/11lGe8Ne5VlvDfsVZbx3rDXWsZ7g7ct4y1Dk9I0zUfPxBC7xn36t/aKpxFY94kksx4u8y0M+BRYn945UbJ8V5mGWBfSvjDR3cgv+UROnC5zg2UmCbJiOF7ASQKc4GKdwJVzqs/mVcN+7sT0CFWqCBU5kd3vZwDpJuB7JlAGDdMOmwlvig2kL6T8/yHUZCFjkMF3xYWqWFEGYhYqY0XpJ/QLPZgNpHWzIN9YbBYw63Po4u+fEN04mVo+AXyLQZDT09jfyZnSuoCGTGps+wRULFmumcieyW+enKxntBrrFd6/id43TyNi/fM7UE3++PdoHN37Cctc6PK6MXg/ZY15oTTICKHLtHTBR26IIoEDVSUQRSYdG75PY9v3L3W+I76LfU+96fqerw9mAffIGy4Mi166FrDnCS4Vgi4RMeSJbhZDHWZfbFkOhrvYUIjgL17y/DSvv7UScl0cNCmPRaYpy8C25R5ni5Vy1VeQGbjXEdiYEf8DNg0ElTdjgDAZsOv2pzSjF0gh0xFk9DRezFtDnITgQIdiHOGh/PlDkRgyLSnviCE3Mf86LyNlmYbLaFCYh8h/F+kJJBHnG74iJl3R6arLnr4k1HxPffxpCiX7GpwUChJyeq9CsJFEvc9sHQ4VNjkByy5mIrY1BfMzDLUow7jvsQ/m5mi8zjZZzMlLjjDeh3IeyxRuqQ9uwoWVbxfKe05S+GUo3oFySBXIBIvMgPTObZuBiRm1NgMWMRptFipqltkMZO/gsRmYiFlhM1B9w71mQGKPsJoIiztSaiqof+TVDNQJp1lNR0YMlJoBjBj/hETtIW7qdxGzm5A6kHU9L25E0gxg50QjBN79MTNPIvJAtDfiApmOz+vs5XGtj8BJ8UnqSotpfIWoLDgY+I8YnPB61MASzhLUtdryWJgH3SzGVkUcg1KhwM+g42sUFu1MOT/R2D+PawIm9qCtCZBTxvk5X5C/qKHeJEFej4vM3dsrwuTnnEPsfxNCenfs4bVkRsbiP24WrxNZ/TRVr15kSv3Tx8ZoRyZoSGZoSaZpSmbkngvF4M9cXSBk4pmoEyX10jTk2bmTXGYzrn9TJ1Fti48kxMDeYMoprBN8xeGGZruhkM/ZUEKwB7lNpjVfZ7PcJiGai8u2Xa/jxTjis1fEEZ93E5/H817NYEE74yJTRVpPJC7yXEgNieWMoBmakcMbGRgc3sgq4vBGu/FGWtIY0VM6Ehb3z+Ei3GwOF/FYLqJdUKa17SE6SiYz9OGBDg80QczhgY6S6fBGhzfyiDi80W680VEyOX8OF2GicriIH9lFtDtBpIhBKbHOEpPqca5PYT8LmdLqIHZndeat7+rMKiymymkWGFFLoO7tNZueQbnXBfD+pVdIHUjwpbqd4JkpGpcPxfJnpPV8Qqxzssmxh+PYw2GEPPZwHHs4jj0cxx6Oexp57OEwoB57ONAQw+WbUSdTUo7GIwbStGGWHAmS5edIkAJAHgnSkSAdCdKRIB0JkhP1SJBQEHenehafTFfMGfyRLRkutWislX7IyVRBx4Idxsdd3+bw1UkMwcxfHSokzBufERUFF9RuhiX1xNPnJ6dnSEThmzn/9PnLcozlEJlvIFiIUsZuLhRXgjPbdKx5UJ2HM9iD7jlGBVM9+85AQWC4zzBBAyzQwneCCALCeQQI/vqJDFYJ/agjS72xwHv0qFUZ44mAi7z5Kt4YcxoeMulDnGCHRPKcsoWZs+g6IS+cc/af+DZl0dGEg+RmwTpPmkO83c6D23AvFOLMNQSR+mSwVd6HAI/elKPaEKs38IesWX2R5ejb+d7Ifigt+u5Zj5PFI9gOgsUsJzCeEOsxo/lY6PlWtByCjEnOXQc9o6+3nNLssYItJlrFegaVVmNDa1CjPFlklcCLkKLQLLPcgOmC71Nm79Dkds48YZl9XOvtIHb0c+Dh8Z/VY0JikaYiI1oQynklfKio73nKWQ6cZfb8c1GiwBlknpDq9WaubnHIRV6vS1d4kduyuZf6b6senvH7WvPjucfdn/tzR+PXTHzjkFwgqeer+I/+pUkCiTMY7ErFimhNuny09RU8nwz6enqMsiNFa6ZzNL1tQ3m4m9rwJmcp0jn6PtzNnavxmXG+yv290ML57WnB3a05l3Fwsp4r3NdVFL3d1QWa5ohBfWcuRrnCFnr+L8gYMk0v1Rj4VrUbda+GKf3+su8b+V/0O0uLlGQjH8y5+Db1jkpIxdvjOeEb7akeqdH24VzwBH3bTqBLx5BZp+NMWDKtZ+PfhDoB7EyZ64WfhsW4BhkKLS9OnKlrKDgJ2rkQZ0rTzNJaQEB10pwzM0kKsCSvLlKX1WJKU7m0nlNa5AshJHBBrb1OhJ1H54L3r7XZGeGE/qc+vsXWSsbcKcuXaIJqKKI95Z/veee8mTWdIZLQf5RSmkhgk7ZpOb+opuuU6ZuEgdnthFI0tpFwojq+Lk915hvWyyBoNFz2CPgotFSLnLOYBkghF7FtWPgJh0gOFpZfbgadlZYpzRaG95lMtRCkFI59tRZkD7OY/lnKG9VBnrBrXN5DQkaB/wOaOD/EYiNAkJpsUZFlNdbkjQJrsCz5ueQ3FL9JZl2hG+yu/38p5Ie47aW5tGMrwg7ufHVHfQTa71ZUQxb7XWjINdVXprS4SOvW/mmSsdLJxPXXBL9UmvR7ZGhgXxtwgAwJGx3fGYh1/uuXlVjnv/66ANlZ0C1YCOY51Gh5OX4q+OvLbZvcBn2mZefkDOFikeYcFo3EuK/fS+C7vd7bZ6stluBMlKf17UBpKd6xcP8MAAD//zT3f3k=" } diff --git a/metricbeat/module/beat/stats/_meta/fields.yml b/metricbeat/module/beat/stats/_meta/fields.yml index 8c1d8973e815..bb48a6c7b651 100644 --- a/metricbeat/module/beat/stats/_meta/fields.yml +++ b/metricbeat/module/beat/stats/_meta/fields.yml @@ -561,10 +561,64 @@ fields: - name: clients type: long - - name: queue.acked - type: long - - name: queue.max_events - type: long + - name: queue + type: group + fields: + - name: acked + type: long + metric_type: counter + description: > + Number of acknowledged events + - name: added.bytes + type: long + metric_type: counter + description: > + Number of bytes added to the queue + - name: added.events + type: long + metric_type: counter + description: > + Number of events added to the queue + - name: consumed.bytes + type: long + metric_type: counter + description: > + Number of bytes consumed from the queue + - name: consumed.events + type: long + metric_type: counter + description: > + Number of events consumed from the queue + - name: filled.bytes + type: long + metric_type: gauge + description: > + Number of bytes filled in the queue + - name: filled.events + type: long + metric_type: gauge + description: > + Number of events filled in the queue + - name: filled.pct + type: float + metric_type: gauge + description: > + Percentage of the queue filled + - name: max_events + type: long + metric_type: gauge + description: > + Maximum number of events allowed in the queue + - name: removed.bytes + type: long + metric_type: counter + description: > + Number of bytes removed from the queue + - name: removed.events + type: long + metric_type: counter + description: > + Number of events removed from the queue - name: events type: group fields: @@ -665,6 +719,22 @@ type: long description: > Number of write errors + - name: latency + type: group + fields: + - name: histogram + type: group + fields: + - name: count + type: long + - name: max + type: float + - name: median + type: long + - name: p95 + type: float + - name: p99 + type: float - name: output type: group fields: diff --git a/metricbeat/module/beat/stats/data.go b/metricbeat/module/beat/stats/data.go index 8cb118669690..b6500903c9e7 100644 --- a/metricbeat/module/beat/stats/data.go +++ b/metricbeat/module/beat/stats/data.go @@ -72,10 +72,13 @@ var ( "bytes": c.Int("bytes"), "errors": c.Int("errors"), "latency": c.Dict("latency", s.Schema{ - "count": c.Int("count"), - "max": c.Int("max"), - "median": c.Float("median"), - "p99": c.Float("p99"), + "histogram": c.Dict("histogram", s.Schema{ + "count": c.Int("count"), + "max": c.Int("max"), + "median": c.Float("median"), + "p95": c.Float("p95"), + "p99": c.Float("p99"), + }), }), }), }), From 04eac62055e90992704e1d45fa580162f01c0d9b Mon Sep 17 00:00:00 2001 From: Mauri de Souza Meneguzzo Date: Fri, 31 Jan 2025 09:41:17 -0300 Subject: [PATCH 06/10] otelconsumer: set document id attribute for elasticsearchexporter (#42412) * otelconsumer: set document id attribute for elasticsearchexporter * add comment about field being removed from elasticsearch --- libbeat/outputs/otelconsumer/otelconsumer.go | 26 +++++++++++++++++++ .../outputs/otelconsumer/otelconsumer_test.go | 21 +++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/libbeat/outputs/otelconsumer/otelconsumer.go b/libbeat/outputs/otelconsumer/otelconsumer.go index 5f1d4951ec1f..2e34d1f60c31 100644 --- a/libbeat/outputs/otelconsumer/otelconsumer.go +++ b/libbeat/outputs/otelconsumer/otelconsumer.go @@ -35,6 +35,11 @@ import ( "go.opentelemetry.io/collector/pdata/plog" ) +const ( + // esDocumentIDAttribute is the attribute key used to store the document ID in the log record. + esDocumentIDAttribute = "elasticsearch.document_id" +) + func init() { outputs.RegisterType("otelconsumer", makeOtelConsumer) } @@ -84,9 +89,30 @@ func (out *otelConsumer) logsPublish(ctx context.Context, batch publisher.Batch) sourceLogs := resourceLogs.ScopeLogs().AppendEmpty() logRecords := sourceLogs.LogRecords() + // Convert the batch of events to Otel plog.Logs. The encoding we + // choose here is to set all fields in a Map in the Body of the log + // record. Each log record encodes a single beats event. + // This way we have full control over the final structure of the log in the + // destination, as long as the exporter allows it. + // For example, the elasticsearchexporter has an encoding specifically for this. + // See https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/35444. events := batch.Events() for _, event := range events { logRecord := logRecords.AppendEmpty() + + if id, ok := event.Content.Meta["_id"]; ok { + // Specify the id as an attribute used by the elasticsearchexporter + // to set the final document ID in Elasticsearch. + // When using the bodymap encoding in the exporter all attributes + // are stripped out of the final Elasticsearch document. + // + // See https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/36882. + switch id := id.(type) { + case string: + logRecord.Attributes().PutStr(esDocumentIDAttribute, id) + } + } + beatEvent := event.Content.Fields.Clone() beatEvent["@timestamp"] = event.Content.Timestamp logRecord.SetTimestamp(pcommon.NewTimestampFromTime(event.Content.Timestamp)) diff --git a/libbeat/outputs/otelconsumer/otelconsumer_test.go b/libbeat/outputs/otelconsumer/otelconsumer_test.go index 094058a2923d..bcb7253023c8 100644 --- a/libbeat/outputs/otelconsumer/otelconsumer_test.go +++ b/libbeat/outputs/otelconsumer/otelconsumer_test.go @@ -43,6 +43,7 @@ func TestPublish(t *testing.T) { event1 := beat.Event{Fields: mapstr.M{"field": 1}} event2 := beat.Event{Fields: mapstr.M{"field": 2}} event3 := beat.Event{Fields: mapstr.M{"field": 3}} + event4 := beat.Event{Meta: mapstr.M{"_id": "abc123"}} makeOtelConsumer := func(t *testing.T, consumeFn func(ctx context.Context, ld plog.Logs) error) *otelConsumer { t.Helper() @@ -118,6 +119,26 @@ func TestPublish(t *testing.T) { assert.Equal(t, outest.BatchRetry, batch.Signals[0].Tag) }) + t.Run("sets the elasticsearchexporter doc id attribute from metadata", func(t *testing.T) { + batch := outest.NewBatch(event4) + + var docID string + otelConsumer := makeOtelConsumer(t, func(ctx context.Context, ld plog.Logs) error { + record := ld.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0) + attr, ok := record.Attributes().Get(esDocumentIDAttribute) + assert.True(t, ok, "document ID attribute should be set") + docID = attr.AsString() + + return nil + }) + + err := otelConsumer.Publish(ctx, batch) + assert.NoError(t, err) + assert.Len(t, batch.Signals, 1) + assert.Equal(t, outest.BatchACK, batch.Signals[0].Tag) + assert.Equal(t, event4.Meta["_id"], docID) + }) + t.Run("sets the @timestamp field with the correct format", func(t *testing.T) { batch := outest.NewBatch(event3) batch.Events()[0].Content.Timestamp = time.Date(2025, time.January, 29, 9, 2, 39, 0, time.UTC) From 76f40863c9918ee1a691a2bb29cbab98a5d83878 Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Fri, 31 Jan 2025 10:48:37 -0500 Subject: [PATCH 07/10] Add status reporting for Journald input (#42462) This commit adds the status reporting for the Journald input. It also adds a debug log to the `UpdateStatus` function from `v2.Context`. --- CHANGELOG.next.asciidoc | 1 + filebeat/input/journald/environment_test.go | 62 ++++++++++++++++++--- filebeat/input/journald/input.go | 16 +++++- filebeat/input/journald/input_test.go | 28 ++++++++++ filebeat/input/v2/input.go | 1 + 5 files changed, 96 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 96576690f1e4..944d8ece4806 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -409,6 +409,7 @@ otherwise no tag is added. {issue}42208[42208] {pull}42403[42403] - Added OAuth2 support with auto token refresh for websocket streaming input. {issue}41989[41989] {pull}42212[42212] - Added infinite & blanket retry options to websockets and improved logging and retry logic. {pull}42225[42225] - Introduce ignore older and start timestamp filters for AWS S3 input. {pull}41804[41804] +- Journald input now can report its status to Elastic-Agent {issue}39791[39791] {pull}42462[42462] *Auditbeat* diff --git a/filebeat/input/journald/environment_test.go b/filebeat/input/journald/environment_test.go index 9ea77d017d15..5b6a8fcf35c2 100644 --- a/filebeat/input/journald/environment_test.go +++ b/filebeat/input/journald/environment_test.go @@ -32,6 +32,7 @@ import ( v2 "github.com/elastic/beats/v7/filebeat/input/v2" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common/acker" + "github.com/elastic/beats/v7/libbeat/management/status" "github.com/elastic/beats/v7/libbeat/statestore" "github.com/elastic/beats/v7/libbeat/statestore/storetest" conf "github.com/elastic/elastic-agent-libs/config" @@ -40,10 +41,11 @@ import ( ) type inputTestingEnvironment struct { - t *testing.T - workingDir string - stateStore *testInputStore - pipeline *mockPipelineConnector + t *testing.T + workingDir string + stateStore *testInputStore + pipeline *mockPipelineConnector + statusReporter *mockStatusReporter pluginInitOnce sync.Once plugin v2.Plugin @@ -54,10 +56,11 @@ type inputTestingEnvironment struct { func newInputTestingEnvironment(t *testing.T) *inputTestingEnvironment { return &inputTestingEnvironment{ - t: t, - workingDir: t.TempDir(), - stateStore: openTestStatestore(), - pipeline: &mockPipelineConnector{}, + t: t, + workingDir: t.TempDir(), + stateStore: openTestStatestore(), + pipeline: &mockPipelineConnector{}, + statusReporter: &mockStatusReporter{}, } } @@ -95,7 +98,7 @@ func (e *inputTestingEnvironment) startInput(ctx context.Context, inp v2.Input) } }() - inputCtx := v2.Context{Logger: logp.L(), Cancelation: ctx} + inputCtx := v2.Context{Logger: logp.L(), Cancelation: ctx, StatusReporter: e.statusReporter} if err := inp.Run(inputCtx, e.pipeline); err != nil { e.t.Errorf("input 'Run' method returned an error: %s", err) } @@ -125,6 +128,25 @@ func (e *inputTestingEnvironment) waitUntilEventCount(count int) { }, 5*time.Second, 10*time.Millisecond, &msg) } +func (e *inputTestingEnvironment) RequireStatuses(expected []statusUpdate) { + t := e.t + t.Helper() + got := e.statusReporter.GetUpdates() + if len(got) != len(expected) { + t.Fatalf("expecting %d updates, got %d", len(expected), len(got)) + } + + for i := range expected { + g, e := got[i], expected[i] + if g != e { + t.Errorf( + "expecting [%d] status update to be {state:%s, msg:%s}, got {state:%s, msg:%s}", + i, e.state.String(), e.msg, g.state.String(), g.msg, + ) + } + } +} + type testInputStore struct { registry *statestore.Registry } @@ -251,3 +273,25 @@ func blockingACKer(starter context.Context) beat.EventListener { } }) } + +type statusUpdate struct { + state status.Status + msg string +} + +type mockStatusReporter struct { + mutex sync.RWMutex + updates []statusUpdate +} + +func (m *mockStatusReporter) UpdateStatus(status status.Status, msg string) { + m.mutex.Lock() + m.updates = append(m.updates, statusUpdate{status, msg}) + m.mutex.Unlock() +} + +func (m *mockStatusReporter) GetUpdates() []statusUpdate { + m.mutex.RLock() + defer m.mutex.RUnlock() + return append([]statusUpdate{}, m.updates...) +} diff --git a/filebeat/input/journald/input.go b/filebeat/input/journald/input.go index e8eb8a7a8773..b83b8098ffae 100644 --- a/filebeat/input/journald/input.go +++ b/filebeat/input/journald/input.go @@ -30,6 +30,7 @@ import ( input "github.com/elastic/beats/v7/filebeat/input/v2" cursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" "github.com/elastic/beats/v7/libbeat/feature" + "github.com/elastic/beats/v7/libbeat/management/status" "github.com/elastic/beats/v7/libbeat/reader" "github.com/elastic/beats/v7/libbeat/reader/parser" conf "github.com/elastic/elastic-agent-libs/config" @@ -155,6 +156,8 @@ func (inp *journald) Run( logger := ctx.Logger. With("path", src.Name()). With("input_id", inp.ID) + + ctx.UpdateStatus(status.Starting, "Starting") currentCheckpoint := initCheckpoint(logger, cursor) mode := inp.Seek @@ -174,7 +177,9 @@ func (inp *journald) Run( journalctl.Factory, ) if err != nil { - return fmt.Errorf("could not start journal reader: %w", err) + wrappedErr := fmt.Errorf("could not start journal reader: %w", err) + ctx.UpdateStatus(status.Failed, wrappedErr.Error()) + return wrappedErr } defer reader.Close() @@ -187,6 +192,7 @@ func (inp *journald) Run( saveRemoteHostname: inp.SaveRemoteHostname, }) + ctx.UpdateStatus(status.Running, "Running") for { entry, err := parser.Next() if err != nil { @@ -198,14 +204,18 @@ func (inp *journald) Run( case errors.Is(err, journalctl.ErrRestarting): continue default: - logger.Errorf("could not read event: %s", err) + msg := fmt.Sprintf("could not read event: %s", err) + ctx.UpdateStatus(status.Failed, msg) + logger.Error(msg) return err } } event := entry.ToEvent() if err := publisher.Publish(event, event.Private); err != nil { - logger.Errorf("could not publish event: %s", err) + msg := fmt.Sprintf("could not publish event: %s", err) + ctx.UpdateStatus(status.Failed, msg) + logger.Errorf(msg) return err } } diff --git a/filebeat/input/journald/input_test.go b/filebeat/input/journald/input_test.go index 5d0edb18cb05..3332ac8150c9 100644 --- a/filebeat/input/journald/input_test.go +++ b/filebeat/input/journald/input_test.go @@ -37,6 +37,7 @@ import ( "github.com/elastic/beats/v7/filebeat/input/journald/pkg/journalfield" v2 "github.com/elastic/beats/v7/filebeat/input/v2" "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/management/status" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" ) @@ -336,6 +337,33 @@ func TestReaderAdapterCanHandleNonStringFields(t *testing.T) { } } +func TestInputCanReportStatus(t *testing.T) { + out := decompress(t, filepath.Join("testdata", "multiple-boots.journal.gz")) + + env := newInputTestingEnvironment(t) + cfg := mapstr.M{ + "paths": []string{out}, + } + inp := env.mustCreateInput(cfg) + + ctx, cancelInput := context.WithCancel(context.Background()) + t.Cleanup(cancelInput) + + env.startInput(ctx, inp) + env.waitUntilEventCount(6) + + env.RequireStatuses([]statusUpdate{ + { + state: status.Starting, + msg: "Starting", + }, + { + state: status.Running, + msg: "Running", + }, + }) +} + func decompress(t *testing.T, namegz string) string { t.Helper() diff --git a/filebeat/input/v2/input.go b/filebeat/input/v2/input.go index f62cc149a936..63824d776301 100644 --- a/filebeat/input/v2/input.go +++ b/filebeat/input/v2/input.go @@ -97,6 +97,7 @@ type Context struct { func (c Context) UpdateStatus(status status.Status, msg string) { if c.StatusReporter != nil { + c.Logger.Debugf("updating status, status: '%s', message: '%s'", status.String(), msg) c.StatusReporter.UpdateStatus(status, msg) } } From 2b8d554d3169807a35e32702833e83935a6adac0 Mon Sep 17 00:00:00 2001 From: William Easton Date: Fri, 31 Jan 2025 14:20:49 -0600 Subject: [PATCH 08/10] Add Kafka output metrics (#42537) --- libbeat/outputs/kafka/config.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/libbeat/outputs/kafka/config.go b/libbeat/outputs/kafka/config.go index 98c7ba065335..bf0b0a86d023 100644 --- a/libbeat/outputs/kafka/config.go +++ b/libbeat/outputs/kafka/config.go @@ -305,11 +305,14 @@ func newSaramaConfig(log *logp.Logger, config *kafkaConfig) (*sarama.Config, err k.Version = version k.Producer.Partitioner = partitioner + k.MetricRegistry = adapter.GetGoMetrics( monitoring.Default, - "libbeat.outputs.kafka", - adapter.Rename("incoming-byte-rate", "bytes_read"), - adapter.Rename("outgoing-byte-rate", "bytes_write"), + "libbeat.outputs", + adapter.Rename("incoming-byte-rate", "read.bytes"), + adapter.Rename("outgoing-byte-rate", "write.bytes"), + adapter.Rename("request-latency-in-ms", "write.latency"), + adapter.Rename("requests-in-flight", "kafka.requests-in-flight"), adapter.GoMetricsNilify, ) From 0e76511316f5ef117d44d034860b9732932064a9 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Mon, 3 Feb 2025 10:15:32 +1100 Subject: [PATCH 09/10] .../input/entityanalytics/provider/okta: Publish events progressively (#42567) Instead of waiting for a full sync (which may take hours) to finish before publishing anything, we now publish data as it is received, so that users have immediate feedback. Incremental updates are also published more frequently: page by page instead of at the end of the pagination sequence. For the full sync, the previous behavior of adding everything to the store and publishing it at the end would theoretically de-duplicate repeated items. However, within a pagination sequence we use opaque cursors from the API that should avoid overlap between pages. Whether an item modified after an earlier page in a sequence can appear again in a later page of the same sequence depends on the API's implementation. Incremental updates begin with a previously seen timestamp value, so there is overlap between updates there, but that is unaffected by this change. For the full sync, markers for the start and end were only published if data is retrieved, but now the markers are published regardless of how much data is received. Some extra checks of configuration are done to decide whether to fetch and publish items of a given type (user or device). --------- Co-authored-by: Dan Kortschak --- CHANGELOG.next.asciidoc | 1 + .../entityanalytics/provider/okta/okta.go | 125 +++++++++--------- .../provider/okta/okta_test.go | 20 ++- 3 files changed, 78 insertions(+), 68 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 944d8ece4806..6fe4669fd840 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -410,6 +410,7 @@ otherwise no tag is added. {issue}42208[42208] {pull}42403[42403] - Added infinite & blanket retry options to websockets and improved logging and retry logic. {pull}42225[42225] - Introduce ignore older and start timestamp filters for AWS S3 input. {pull}41804[41804] - Journald input now can report its status to Elastic-Agent {issue}39791[39791] {pull}42462[42462] +- Publish events progressively in the Okta provider of the Entity Analytics input. {issue}40106[40106] {pull}42567[42567] *Auditbeat* diff --git a/x-pack/filebeat/input/entityanalytics/provider/okta/okta.go b/x-pack/filebeat/input/entityanalytics/provider/okta/okta.go index 2549d6593578..1ed7b4c6ee9a 100644 --- a/x-pack/filebeat/input/entityanalytics/provider/okta/okta.go +++ b/x-pack/filebeat/input/entityanalytics/provider/okta/okta.go @@ -315,32 +315,31 @@ func (p *oktaInput) runFullSync(inputCtx v2.Context, store *kvstore.Store, clien } }() - ctx := ctxtool.FromCanceller(inputCtx.Cancelation) - p.logger.Debugf("Starting fetch...") - _, err = p.doFetchUsers(ctx, state, true) - if err != nil { - return err - } - _, err = p.doFetchDevices(ctx, state, true) - if err != nil { - return err - } - wantUsers := p.cfg.wantUsers() wantDevices := p.cfg.wantDevices() - if (len(state.users) != 0 && wantUsers) || (len(state.devices) != 0 && wantDevices) { + if wantUsers || wantDevices { + ctx := ctxtool.FromCanceller(inputCtx.Cancelation) + p.logger.Debugf("Starting fetch...") + tracker := kvstore.NewTxTracker(ctx) start := time.Now() p.publishMarker(start, start, inputCtx.ID, true, client, tracker) + if wantUsers { - for _, u := range state.users { + err = p.doFetchUsers(ctx, state, true, func(u *User) { p.publishUser(u, state, inputCtx.ID, client, tracker) + }) + if err != nil { + return err } } if wantDevices { - for _, d := range state.devices { + err = p.doFetchDevices(ctx, state, true, func(d *Device) { p.publishDevice(d, state, inputCtx.ID, client, tracker) + }) + if err != nil { + return err } } @@ -348,10 +347,10 @@ func (p *oktaInput) runFullSync(inputCtx v2.Context, store *kvstore.Store, clien p.publishMarker(end, end, inputCtx.ID, false, client, tracker) tracker.Wait() - } - if ctx.Err() != nil { - return ctx.Err() + if ctx.Err() != nil { + return ctx.Err() + } } state.lastSync = time.Now() @@ -381,27 +380,28 @@ func (p *oktaInput) runIncrementalUpdate(inputCtx v2.Context, store *kvstore.Sto }() ctx := ctxtool.FromCanceller(inputCtx.Cancelation) - updatedUsers, err := p.doFetchUsers(ctx, state, false) - if err != nil { - return err - } - updatedDevices, err := p.doFetchDevices(ctx, state, false) - if err != nil { - return err - } + tracker := kvstore.NewTxTracker(ctx) - var tracker *kvstore.TxTracker - if len(updatedUsers) != 0 || len(updatedDevices) != 0 { - tracker = kvstore.NewTxTracker(ctx) - for _, u := range updatedUsers { + if p.cfg.wantUsers() { + p.logger.Debugf("Fetching changed users...") + err = p.doFetchUsers(ctx, state, false, func(u *User) { p.publishUser(u, state, inputCtx.ID, client, tracker) + }) + if err != nil { + return err } - for _, d := range updatedDevices { + } + if p.cfg.wantDevices() { + p.logger.Debugf("Fetching changed devices...") + err = p.doFetchDevices(ctx, state, false, func(d *Device) { p.publishDevice(d, state, inputCtx.ID, client, tracker) + }) + if err != nil { + return err } - tracker.Wait() } + tracker.Wait() if ctx.Err() != nil { return ctx.Err() } @@ -417,10 +417,10 @@ func (p *oktaInput) runIncrementalUpdate(inputCtx v2.Context, store *kvstore.Sto // doFetchUsers handles fetching user identities from Okta. If fullSync is true, then // any existing deltaLink will be ignored, forcing a full synchronization from Okta. // Returns a set of modified users by ID. -func (p *oktaInput) doFetchUsers(ctx context.Context, state *stateStore, fullSync bool) ([]*User, error) { +func (p *oktaInput) doFetchUsers(ctx context.Context, state *stateStore, fullSync bool, publish func(u *User)) error { if !p.cfg.wantUsers() { p.logger.Debugf("Skipping user collection from API: dataset=%s", p.cfg.Dataset) - return nil, nil + return nil } var ( @@ -446,29 +446,29 @@ func (p *oktaInput) doFetchUsers(ctx context.Context, state *stateStore, fullSyn const omit = okta.OmitCredentials | okta.OmitCredentialsLinks | okta.OmitTransitioningToStatus var ( - users []*User + n int lastUpdated time.Time ) for { batch, h, err := okta.GetUserDetails(ctx, p.client, p.cfg.OktaDomain, p.cfg.OktaToken, "", query, omit, p.lim, p.logger) if err != nil { - p.logger.Debugf("received %d users from API", len(users)) - return nil, err + p.logger.Debugf("received %d users from API", n) + return err } p.logger.Debugf("received batch of %d users from API", len(batch)) if fullSync { for _, u := range batch { - p.addUserMetadata(ctx, u, state) + publish(p.addUserMetadata(ctx, u, state)) if u.LastUpdated.After(lastUpdated) { lastUpdated = u.LastUpdated } } } else { - users = grow(users, len(batch)) for _, u := range batch { su := p.addUserMetadata(ctx, u, state) - users = append(users, su) + publish(su) + n++ if u.LastUpdated.After(lastUpdated) { lastUpdated = u.LastUpdated } @@ -480,8 +480,8 @@ func (p *oktaInput) doFetchUsers(ctx context.Context, state *stateStore, fullSyn if err == io.EOF { break } - p.logger.Debugf("received %d users from API", len(users)) - return users, err + p.logger.Debugf("received %d users from API", n) + return err } query = next } @@ -495,8 +495,8 @@ func (p *oktaInput) doFetchUsers(ctx context.Context, state *stateStore, fullSyn query.Add("search", fmt.Sprintf(`lastUpdated ge "%s" and status pr`, lastUpdated.Format(okta.ISO8601))) state.nextUsers = query.Encode() - p.logger.Debugf("received %d users from API", len(users)) - return users, nil + p.logger.Debugf("received %d users from API", n) + return nil } func (p *oktaInput) addUserMetadata(ctx context.Context, u okta.User, state *stateStore) *User { @@ -541,10 +541,10 @@ func (p *oktaInput) addUserMetadata(ctx context.Context, u okta.User, state *sta // If fullSync is true, then any existing deltaLink will be ignored, forcing a full // synchronization from Okta. // Returns a set of modified devices by ID. -func (p *oktaInput) doFetchDevices(ctx context.Context, state *stateStore, fullSync bool) ([]*Device, error) { +func (p *oktaInput) doFetchDevices(ctx context.Context, state *stateStore, fullSync bool, publish func(d *Device)) error { if !p.cfg.wantDevices() { p.logger.Debugf("Skipping device collection from API: dataset=%s", p.cfg.Dataset) - return nil, nil + return nil } var ( @@ -575,14 +575,14 @@ func (p *oktaInput) doFetchDevices(ctx context.Context, state *stateStore, fullS userQueryInit = cloneURLValues(deviceQuery) var ( - devices []*Device + n int lastUpdated time.Time ) for { batch, h, err := okta.GetDeviceDetails(ctx, p.client, p.cfg.OktaDomain, p.cfg.OktaToken, "", deviceQuery, p.lim, p.logger) if err != nil { - p.logger.Debugf("received %d devices from API", len(devices)) - return nil, err + p.logger.Debugf("received %d devices from API", n) + return err } p.logger.Debugf("received batch of %d devices from API", len(batch)) @@ -601,7 +601,7 @@ func (p *oktaInput) doFetchDevices(ctx context.Context, state *stateStore, fullS users, h, err := okta.GetDeviceUsers(ctx, p.client, p.cfg.OktaDomain, p.cfg.OktaToken, d.ID, userQuery, omit, p.lim, p.logger) if err != nil { p.logger.Debugf("received %d device users from API", len(users)) - return nil, err + return err } p.logger.Debugf("received batch of %d device users from API", len(users)) @@ -617,8 +617,8 @@ func (p *oktaInput) doFetchDevices(ctx context.Context, state *stateStore, fullS if err == io.EOF { break } - p.logger.Debugf("received %d devices from API", len(devices)) - return devices, err + p.logger.Debugf("received %d devices from API", n) + return err } userQuery = next } @@ -626,15 +626,16 @@ func (p *oktaInput) doFetchDevices(ctx context.Context, state *stateStore, fullS if fullSync { for _, d := range batch { - state.storeDevice(d) + publish(state.storeDevice(d)) if d.LastUpdated.After(lastUpdated) { lastUpdated = d.LastUpdated } } } else { - devices = grow(devices, len(batch)) for _, d := range batch { - devices = append(devices, state.storeDevice(d)) + sd := state.storeDevice(d) + publish(sd) + n++ if d.LastUpdated.After(lastUpdated) { lastUpdated = d.LastUpdated } @@ -646,8 +647,8 @@ func (p *oktaInput) doFetchDevices(ctx context.Context, state *stateStore, fullS if err == io.EOF { break } - p.logger.Debugf("received %d devices from API", len(devices)) - return devices, err + p.logger.Debugf("received %d devices from API", n) + return err } deviceQuery = next } @@ -661,8 +662,8 @@ func (p *oktaInput) doFetchDevices(ctx context.Context, state *stateStore, fullS deviceQuery.Add("search", fmt.Sprintf(`lastUpdated ge "%s" and status pr`, lastUpdated.Format(okta.ISO8601))) state.nextDevices = deviceQuery.Encode() - p.logger.Debugf("received %d devices from API", len(devices)) - return devices, nil + p.logger.Debugf("received %d devices from API", n) + return nil } func cloneURLValues(a url.Values) url.Values { @@ -677,14 +678,6 @@ type entity interface { *User | *Device | okta.User } -func grow[T entity](e []T, n int) []T { - if len(e)+n <= cap(e) { - return e - } - new := append(e, make([]T, n)...) - return new[:len(e)] -} - // publishMarker will publish a write marker document using the given beat.Client. // If start is true, then it will be a start marker, otherwise an end marker. func (p *oktaInput) publishMarker(ts, eventTime time.Time, inputID string, start bool, client beat.Client, tracker *kvstore.TxTracker) { diff --git a/x-pack/filebeat/input/entityanalytics/provider/okta/okta_test.go b/x-pack/filebeat/input/entityanalytics/provider/okta/okta_test.go index ea2a710d8c51..f7bc7c45d799 100644 --- a/x-pack/filebeat/input/entityanalytics/provider/okta/okta_test.go +++ b/x-pack/filebeat/input/entityanalytics/provider/okta/okta_test.go @@ -211,8 +211,13 @@ func TestOktaDoFetch(t *testing.T) { t.Run("users", func(t *testing.T) { n = 0 + var got []*User + published := make(map[string]struct{}) - got, err := a.doFetchUsers(ctx, ss, false) + err := a.doFetchUsers(ctx, ss, false, func(u *User) { + got = append(got, u) + published[u.ID] = struct{}{} + }) if err != nil { t.Fatalf("unexpected error from doFetch: %v", err) } @@ -220,6 +225,9 @@ func TestOktaDoFetch(t *testing.T) { if len(got) != wantCount(repeats, test.wantUsers) { t.Errorf("unexpected number of results: got:%d want:%d", len(got), wantCount(repeats, test.wantUsers)) } + if len(published) != len(got) { + t.Errorf("unexpected number of distinct users published: got:%d want:%d", len(published), len(got)) + } for i, g := range got { wantID := fmt.Sprintf("userid%d", i+1) if g.ID != wantID { @@ -244,8 +252,13 @@ func TestOktaDoFetch(t *testing.T) { t.Run("devices", func(t *testing.T) { n = 0 + var got []*Device + published := make(map[string]struct{}) - got, err := a.doFetchDevices(ctx, ss, false) + err := a.doFetchDevices(ctx, ss, false, func(d *Device) { + got = append(got, d) + published[d.ID] = struct{}{} + }) if err != nil { t.Fatalf("unexpected error from doFetch: %v", err) } @@ -253,6 +266,9 @@ func TestOktaDoFetch(t *testing.T) { if len(got) != wantCount(repeats, test.wantDevices) { t.Errorf("unexpected number of results: got:%d want:%d", len(got), wantCount(repeats, test.wantDevices)) } + if len(published) != len(got) { + t.Errorf("unexpected number of distinct devices published: got:%d want:%d", len(published), len(got)) + } for i, g := range got { if wantID := fmt.Sprintf("deviceid%d", i+1); g.ID != wantID { t.Errorf("unexpected device ID for device %d: got:%s want:%s", i, g.ID, wantID) From 9134522bc46ce54151d2b0473f56ca0786cfef0c Mon Sep 17 00:00:00 2001 From: Carson Ip Date: Mon, 3 Feb 2025 20:41:39 +0000 Subject: [PATCH 10/10] metricbeat: Update beat module with apm-server tail sampling monitoring metrics fields (#42569) Update metricbeat beat module mapping to add back missing apm-server tail sampling montioring metrics fields. The fields were missed in #40127 --- CHANGELOG.next.asciidoc | 1 + metricbeat/docs/fields.asciidoc | 152 ++++++++++++++++++ metricbeat/module/beat/_meta/fields.yml | 45 +++++- metricbeat/module/beat/fields.go | 2 +- metricbeat/module/beat/stats/_meta/fields.yml | 34 +++- .../_meta/test/apm-server.stats.712.json | 115 +++++++------ 6 files changed, 293 insertions(+), 56 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 6fe4669fd840..41088cb58bd4 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -470,6 +470,7 @@ otherwise no tag is added. {issue}42208[42208] {pull}42403[42403] - Preserve queries for debugging when `merge_results: true` in SQL module {pull}42271[42271] - Collect more fields from ES node/stats metrics and only those that are necessary {pull}42421[42421] - Add new metricset wmi for the windows module. {pull}42017[42017] +- Update beat module with apm-server tail sampling monitoring metrics fields {pull}42569[42569] *Metricbeat* - Add benchmark module {pull}41801[41801] diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 3d571d94d63a..c09e2a5bb87c 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -7573,6 +7573,91 @@ alias to: beat.stats.apm_server.root.unset -- + + +*`beats_stats.apm-server.sampling.tail.dynamic_service_groups`*:: ++ +-- +type: alias + +alias to: beat.stats.apm_server.sampling.tail.dynamic_service_groups + +-- + + +*`beats_stats.apm-server.sampling.tail.events.dropped`*:: ++ +-- +type: alias + +alias to: beat.stats.apm_server.sampling.tail.events.dropped + +-- + +*`beats_stats.apm-server.sampling.tail.events.failed_writes`*:: ++ +-- +type: alias + +alias to: beat.stats.apm_server.sampling.tail.events.failed_writes + +-- + +*`beats_stats.apm-server.sampling.tail.events.head_unsampled`*:: ++ +-- +type: alias + +alias to: beat.stats.apm_server.sampling.tail.events.head_unsampled + +-- + +*`beats_stats.apm-server.sampling.tail.events.processed`*:: ++ +-- +type: alias + +alias to: beat.stats.apm_server.sampling.tail.events.processed + +-- + +*`beats_stats.apm-server.sampling.tail.events.sampled`*:: ++ +-- +type: alias + +alias to: beat.stats.apm_server.sampling.tail.events.sampled + +-- + +*`beats_stats.apm-server.sampling.tail.events.stored`*:: ++ +-- +type: alias + +alias to: beat.stats.apm_server.sampling.tail.events.stored + +-- + + +*`beats_stats.apm-server.sampling.tail.storage.lsm_size`*:: ++ +-- +type: alias + +alias to: beat.stats.apm_server.sampling.tail.storage.lsm_size + +-- + +*`beats_stats.apm-server.sampling.tail.storage.value_log_size`*:: ++ +-- +type: alias + +alias to: beat.stats.apm_server.sampling.tail.storage.value_log_size + +-- + *`beats_stats.apm-server.sampling.transactions_dropped`*:: + -- @@ -9663,6 +9748,73 @@ type: long -- + + +*`beat.stats.apm_server.sampling.tail.dynamic_service_groups`*:: ++ +-- +type: long + +-- + + +*`beat.stats.apm_server.sampling.tail.events.dropped`*:: ++ +-- +type: long + +-- + +*`beat.stats.apm_server.sampling.tail.events.failed_writes`*:: ++ +-- +type: long + +-- + +*`beat.stats.apm_server.sampling.tail.events.head_unsampled`*:: ++ +-- +type: long + +-- + +*`beat.stats.apm_server.sampling.tail.events.processed`*:: ++ +-- +type: long + +-- + +*`beat.stats.apm_server.sampling.tail.events.sampled`*:: ++ +-- +type: long + +-- + +*`beat.stats.apm_server.sampling.tail.events.stored`*:: ++ +-- +type: long + +-- + + +*`beat.stats.apm_server.sampling.tail.storage.lsm_size`*:: ++ +-- +type: long + +-- + +*`beat.stats.apm_server.sampling.tail.storage.value_log_size`*:: ++ +-- +type: long + +-- + *`beat.stats.apm_server.sampling.transactions_dropped`*:: + -- diff --git a/metricbeat/module/beat/_meta/fields.yml b/metricbeat/module/beat/_meta/fields.yml index b4ba9ae25890..25a859aba8be 100644 --- a/metricbeat/module/beat/_meta/fields.yml +++ b/metricbeat/module/beat/_meta/fields.yml @@ -477,9 +477,48 @@ - name: unset type: alias path: beat.stats.apm_server.root.unset - - name: sampling.transactions_dropped - type: alias - path: beat.stats.apm_server.sampling.transactions_dropped + - name: sampling + type: group + fields: + - name: tail + type: group + fields: + - name: dynamic_service_groups + type: alias + path: beat.stats.apm_server.sampling.tail.dynamic_service_groups + - name: events + type: group + fields: + - name: dropped + type: alias + path: beat.stats.apm_server.sampling.tail.events.dropped + - name: failed_writes + type: alias + path: beat.stats.apm_server.sampling.tail.events.failed_writes + - name: head_unsampled + type: alias + path: beat.stats.apm_server.sampling.tail.events.head_unsampled + - name: processed + type: alias + path: beat.stats.apm_server.sampling.tail.events.processed + - name: sampled + type: alias + path: beat.stats.apm_server.sampling.tail.events.sampled + - name: stored + type: alias + path: beat.stats.apm_server.sampling.tail.events.stored + - name: storage + type: group + fields: + - name: lsm_size + type: alias + path: beat.stats.apm_server.sampling.tail.storage.lsm_size + - name: value_log_size + type: alias + path: beat.stats.apm_server.sampling.tail.storage.value_log_size + - name: transactions_dropped + type: alias + path: beat.stats.apm_server.sampling.transactions_dropped - name: server type: group fields: diff --git a/metricbeat/module/beat/fields.go b/metricbeat/module/beat/fields.go index 45e1586c4e26..146a920a0289 100644 --- a/metricbeat/module/beat/fields.go +++ b/metricbeat/module/beat/fields.go @@ -32,5 +32,5 @@ func init() { // AssetBeat returns asset data. // This is the base64 encoded zlib format compressed contents of module/beat. func AssetBeat() string { - return "eJzsXUuPpLiW3tevsHI9jVSLWnQuekatmZF6MQ+NWprF1VXKASci3GkwZZusyvvrr4CA4OHHAQxJVJGbblUm3/nOAc4L+/gX8grvz+QEVH8iRDPN4Zk8/Q5UP30iJAEVS5ZrJrJn8tsnQggpf0VSkRQcPhEigQNV8Ewu9BMhCrRm2UU9k789KcWf/oU8XbXOn/5e/u4qpH6JRXZml2dyplyV158Z8EQ9V8i/kIymUHNRL0pTrap/J0S/56UIKYr89i/d67rX0jz9RYF8A9n+ynS5CaIHE6e9f7dh2HC6WBK+FqB0FIsi06O/apApZ1QZfptTfa1NElUWiWievtQaRjROIzf6nYPKRVZZ3CzepJhLuS64TTG/cjgFa+pW+92JgJRC2uS49fTp2tOXCwWJ9c8wSk9SvFYr8ojF3I0VyTmlNtwSiEViegjXJecR27A7C3liSQLZ5gT9khuOLNMgM8o3p+gVfGf4RjlLvhYg3z+AJUJ4wzQFfRXbv8cesQ27TOizKLLt+XkFNwy/FlBs/zK7pbYBj2rgLGXbe0K/5IajZimIYnuGPrktPyE4lZftb7FXcMOwyGihr0Kyf3xAPEYJ7zB9o4zTE9/enBjZDc/KfVK9PUmn4B671fM7GseQ6y2fqEqvyCv3g1K8mh0uw8uETkXCzmx782FENzTF69bsDBLvvkFB+MJwjNo+3xfIdHy+BKtwgVOlWayAyvi6TolJ4ytEkGnJwF1sLCw4b6aJeipFWPF9uhLOEtQ1OlPGCwnuwnQVxigGZtKqiGNQ6iNZuym0VRvo+Bpty9Mps0/sTDk/0djmb1ak55TcJ3krm7bn6BLcp+hPYlakaRPeUPyLwqXXcSSL/OlF5vFKnTrOIXa7zhAZlK/l2Ze4JNLWlo9Ki0U37Twt0TFXa2u0T9NlGIIwDpmQwRG0ccgsA3lbqkPCztZqn7HPTgRpKzLRXmSSzcgSu6GankMdMC2Jj9EDzwzfuvgYTbC8JjYQPkYZNLleeYx6KjfwLT4yDWlF05yzbFiN9JmGCErwBlnJLwb2Bn4zhY1NjZbRJBYfG09bzkdA9VtoZkT9oLd1TBxNZw/OZkzfxqZt+Gg+fNR2mJ9zcVn/8/V2zqQ0en3DSsUON+K0zZGSk5kme6Bs3KPCYyTiHiUeJgf36LH79NvGH5t5p6AlizdYLyUyVaQgoyJTRZ4LqSF5SaTI85U/Et0NdFM1mkXlIwNnQ/yInR7zHOGTzLfaQ0ZQmxaPFkRtejxgHLWp8kChdKQCNppqSWPPV8hHrd5q1Y4Y5LHOEYLIbKM9ZASyKPFoAciixgPGH4smDxR+hhq4mDRsr1qbXMPRjjRZuTTW0Y702+YIZmSmyR4tlNlVeKBAZlfiscKYXY/HCGIG/kc7cmSgR2xH9ogfsdNjniN8kvlWe8gI+gO0I516PGAcfex2pFmFn7sdWdnkaEeirHOEIDLbaA8ZgR6/HelS4wHjz0O3I40a+JZZ5lLEoJQItxeqslnJI1NnIVOqmchMus3fLNtyjnCy+vXqRtSQwtpF9TnNNmKGEtXy0hLocLQWCdKs9uyeX7Y7sKNupYF7r/7G86jcuzdxyk82QH/gjpfjJrNEbCStwjvJcqZoXD67G701WIltbiHE0PfP96nrTaQraf7YI+luGv6EM+n6mu9tKJ2Z3V6m0hnZ7WosnZHhzubSGTnubzCdhea+JtMZSe5pNJ2R4I5m0xn57Wk4nZHgvqbTGSnuajydmeHu5tMZae5yQJ2N6b4m1BlZ/mwj6vpG2N2MOhO9fQ2pMzHc0ZQ6E71Nx9RVBOxz6tpd3p1iWVmWR9hJuAjgJbSchrPlyU4r8cbEP3It3ur4E1bjQ933Vo/b+O2lIrfw21VNbuG4s6rcwnJ/dbmV6L4qcwvNPdXmFoo7qs4tDPdUn1so7qtCt5DcVY1u47i7Kt1CdJd1up3rvip1C8+frVYfmmF31bqZ4L7qdTPHHVXsZoKb1uy3//SBG3Gn+hS5vpQp565dhTJ/5kZV9iw7i2gE0T5FNB26g6nYI4g2zr3nS7FHEO09LEZOair2CKL1gCAVE8NCYir8EMW+o2f6E6HelYZwh/HFeRHFwjwKf+JLUTOL7Ij3bbQ0iT4HE2iB60n7ElbaGK6vW2BxBryevEzIddaQPX1+Cteo6mpUMjY+Al3ZX1YVbrpHHemrCv9ifLU5Ow0CBln0bucsB86ylbqYnEGm3a3DSUa6aR81rCOXhF7hHNH4NeQ6xxETn5S2m/oGns5umPxZs7cVioiR2pU6KvLIa3uUa+0YtfHC7gs9U8a35OWR19JiXIPclJhPYus4ihNn6rolN7/I+7cZvUZf1EbMLe7e19FrdJVtpMzi2qKr0LmxF7Xc69fyN3BxdodOlpuzNtDdv7mEbeV1h5xQPvdEdXy17moMzsonbrNQ0KeFDQRJkXMW01VaTxZmjUS/zdaOUn1iyBilhUhptqK37bPyidvI2w5J2YTdIxJNotO7/S4vyEFvXDwiekwQ3/QXU3HIcLSAwpGwgjfSv0mmYeWb4pPR57LybXEK6Yx6OTPTyTLLM4NUJAWHSBZZ5ju8ZpaaNXUn/oCK0lSuUaDemDjgR0REviaPIbql+0wWNROqxibkV0hBUv5i/E40scXlh+wJL3LNUojSAK1KD17bfadZwh2t0UWzBFnKdHSlMmDj4sY38mD3KShxDrgarU/Bit2WKrl11c4C4UbUexM+rf56lbt6iV8y+B7Qng3byIXcUU3I9xfKuTCdx7OUQg0f2eEHPFxpWgAeiMRMBXS7rXgTahtgbQ9PgIZrXqxed6+xabo2SfVZCLEvOj6rKAfJRBIVK9S1HTI4SV1eXwuh6Ra0vILu4wPMjqxPJ+QEn9piLpkEbQbiMUX9DxiJbc1xlUJrDgnqqrA8p8keMzZurV+frEVsx+vQONb2V3eGJ72zcUEPGVTu3m6l5TycAvqhbQ1zuJE78m+5VeDats8CI6ZLqVD0ErzcHlNyiXHHyeXx9/YRV7P4NaSO5SvpQx5RsJVBIUjYsXvtt+iNcuvS4ZkcfMB9AivcCR/wkMAK98EP3a6DUiDXMIIHdyB+BRP0kD8N5Y6+bk1fQwWcKs1iBVTG12AdklPBX19u+8vWqS59S41n2LvpdXctEvU0idxS763FNOcQdN4Ugptd6r11lMB3MDZeA9wQ12e6UBo3Grg+CrZ3QYLj+1ZwRi5x971rSkvxvh2pvsBuM1S9lNc3Fhzf/uEt38UyXoiq/11lGe8Ne5VlvDfsVZbx3rDXWsZ7g7ct4y1Dk9I0zUfPxBC7xn36t/aKpxFY94kksx4u8y0M+BRYn945UbJ8V5mGWBfSvjDR3cgv+UROnC5zg2UmCbJiOF7ASQKc4GKdwJVzqs/mVcN+7sT0CFWqCBU5kd3vZwDpJuB7JlAGDdMOmwlvig2kL6T8/yHUZCFjkMF3xYWqWFEGYhYqY0XpJ/QLPZgNpHWzIN9YbBYw63Po4u+fEN04mVo+AXyLQZDT09jfyZnSuoCGTGps+wRULFmumcieyW+enKxntBrrFd6/id43TyNi/fM7UE3++PdoHN37Cctc6PK6MXg/ZY15oTTICKHLtHTBR26IIoEDVSUQRSYdG75PY9v3L3W+I76LfU+96fqerw9mAffIGy4Mi166FrDnCS4Vgi4RMeSJbhZDHWZfbFkOhrvYUIjgL17y/DSvv7UScl0cNCmPRaYpy8C25R5ni5Vy1VeQGbjXEdiYEf8DNg0ElTdjgDAZsOv2pzSjF0gh0xFk9DRezFtDnITgQIdiHOGh/PlDkRgyLSnviCE3Mf86LyNlmYbLaFCYh8h/F+kJJBHnG74iJl3R6arLnr4k1HxPffxpCiX7GpwUChJyeq9CsJFEvc9sHQ4VNjkByy5mIrY1BfMzDLUow7jvsQ/m5mi8zjZZzMlLjjDeh3IeyxRuqQ9uwoWVbxfKe05S+GUo3oFySBXIBIvMgPTObZuBiRm1NgMWMRptFipqltkMZO/gsRmYiFlhM1B9w71mQGKPsJoIiztSaiqof+TVDNQJp1lNR0YMlJoBjBj/hETtIW7qdxGzm5A6kHU9L25E0gxg50QjBN79MTNPIvJAtDfiApmOz+vs5XGtj8BJ8UnqSotpfIWoLDgY+I8YnPB61MASzhLUtdryWJgH3SzGVkUcg1KhwM+g42sUFu1MOT/R2D+PawIm9qCtCZBTxvk5X5C/qKHeJEFej4vM3dsrwuTnnEPsfxNCenfs4bVkRsbiP24WrxNZ/TRVr15kSv3Tx8ZoRyZoSGZoSaZpSmbkngvF4M9cXSBk4pmoEyX10jTk2bmTXGYzrn9TJ1Fti48kxMDeYMoprBN8xeGGZruhkM/ZUEKwB7lNpjVfZ7PcJiGai8u2Xa/jxTjis1fEEZ93E5/H817NYEE74yJTRVpPJC7yXEgNieWMoBmakcMbGRgc3sgq4vBGu/FGWtIY0VM6Ehb3z+Ei3GwOF/FYLqJdUKa17SE6SiYz9OGBDg80QczhgY6S6fBGhzfyiDi80W680VEyOX8OF2GicriIH9lFtDtBpIhBKbHOEpPqca5PYT8LmdLqIHZndeat7+rMKiymymkWGFFLoO7tNZueQbnXBfD+pVdIHUjwpbqd4JkpGpcPxfJnpPV8Qqxzssmxh+PYw2GEPPZwHHs4jj0cxx6Oexp57OEwoB57ONAQw+WbUSdTUo7GIwbStGGWHAmS5edIkAJAHgnSkSAdCdKRIB0JkhP1SJBQEHenehafTFfMGfyRLRkutWislX7IyVRBx4Idxsdd3+bw1UkMwcxfHSokzBufERUFF9RuhiX1xNPnJ6dnSEThmzn/9PnLcozlEJlvIFiIUsZuLhRXgjPbdKx5UJ2HM9iD7jlGBVM9+85AQWC4zzBBAyzQwneCCALCeQQI/vqJDFYJ/agjS72xwHv0qFUZ44mAi7z5Kt4YcxoeMulDnGCHRPKcsoWZs+g6IS+cc/af+DZl0dGEg+RmwTpPmkO83c6D23AvFOLMNQSR+mSwVd6HAI/elKPaEKs38IesWX2R5ejb+d7Ifigt+u5Zj5PFI9gOgsUsJzCeEOsxo/lY6PlWtByCjEnOXQc9o6+3nNLssYItJlrFegaVVmNDa1CjPFlklcCLkKLQLLPcgOmC71Nm79Dkds48YZl9XOvtIHb0c+Dh8Z/VY0JikaYiI1oQynklfKio73nKWQ6cZfb8c1GiwBlknpDq9WaubnHIRV6vS1d4kduyuZf6b6senvH7WvPjucfdn/tzR+PXTHzjkFwgqeer+I/+pUkCiTMY7ErFimhNuny09RU8nwz6enqMsiNFa6ZzNL1tQ3m4m9rwJmcp0jn6PtzNnavxmXG+yv290ML57WnB3a05l3Fwsp4r3NdVFL3d1QWa5ohBfWcuRrnCFnr+L8gYMk0v1Rj4VrUbda+GKf3+su8b+V/0O0uLlGQjH8y5+Db1jkpIxdvjOeEb7akeqdH24VzwBH3bTqBLx5BZp+NMWDKtZ+PfhDoB7EyZ64WfhsW4BhkKLS9OnKlrKDgJ2rkQZ0rTzNJaQEB10pwzM0kKsCSvLlKX1WJKU7m0nlNa5AshJHBBrb1OhJ1H54L3r7XZGeGE/qc+vsXWSsbcKcuXaIJqKKI95Z/veee8mTWdIZLQf5RSmkhgk7ZpOb+opuuU6ZuEgdnthFI0tpFwojq+Lk915hvWyyBoNFz2CPgotFSLnLOYBkghF7FtWPgJh0gOFpZfbgadlZYpzRaG95lMtRCkFI59tRZkD7OY/lnKG9VBnrBrXN5DQkaB/wOaOD/EYiNAkJpsUZFlNdbkjQJrsCz5ueQ3FL9JZl2hG+yu/38p5Ie47aW5tGMrwg7ufHVHfQTa71ZUQxb7XWjINdVXprS4SOvW/mmSsdLJxPXXBL9UmvR7ZGhgXxtwgAwJGx3fGYh1/uuXlVjnv/66ANlZ0C1YCOY51Gh5OX4q+OvLbZvcBn2mZefkDOFikeYcFo3EuK/fS+C7vd7bZ6stluBMlKf17UBpKd6xcP8MAAD//zT3f3k=" + return "eJzsXUuP5DhyvvevIOrsETCHPkwd1sbCNrAHP2As4INhJJgSM5NTlKghqequ+fULSSmlHnwEJVKl7FZdZtBV+uKLIBVBhsiIX9Ab+XhFZ4LVF4QUVYy8ope/EqxeviCUEZkKWirKi1f0ly8IIVT/CuU8qxj5gpAgjGBJXtEVf0FIEqVocZWv6P9epGQv/4RebkqVL/9f/+7GhTqlvLjQ6yu6YCbr5y+UsEy+Nsi/oALnpOUiT1JhJZt/R0h9lLUIwavy/i/D54bP4jL/RRLxTkT/K93jOogRTJqP/t2EYcIZYgnyR0WkSlJeFWr2Vx0yZhRLzW9LrG6tSZLGIgku81OrYYLTPLGjPzjIkheNxfXidYrZlBuCmxRzKwdTsKVutN+DCBGCC5Mcu54uXUf6Mi5JZvwziNJeirdqJQ6xkNGISM4qteOWkZRnukkYl5xDbMfuwsWZZhkpNifoltxxpIUiosBsc4pOwQ+G75jR7I+KiI9PYAkQ3jHNibrx7d9jh9iOXcHVhVfF9vycgjuGf1Sk2v5ltkvtAx5WhNGcbu8J3ZI7jormhFfbM3TJ7flxzrC4bj/ETsEdw6rAlbpxQf/8hHgMEj5g+o4pw2e2vTkhsjuejfvEanuSVsEjdtHXdzhNSam2nFGNXolT7ict8Vp2sBVewVXOM3qh25sPIrqjyd+2ZqeR+PANkoTfGM5R+/l9JYVKL9dgO1zCsFQ0lQSL9BZni4nTG0lIoQQl9s3Gyg3n3TTJSKUEKn5MV5CLIPKWXDBllSD2jWkUxiAGetKySlMi5WeytlPod21EpbdkW55WmWNiF8zYGacmfxORnlXymOR927Q9R5vgMUX3IiYiTZPwjuLvmFxHGUe0yp9eRZlGytQxRlK76wyxgnKlPMcS10Ta1vJJbbHkrp0jJTrnakyNjmnaDIMAxkEeKzgENg5aZCBnSnVK2JpaHTN22QkBbYU87YW8bIbW2A2U9JzqAElJfI4ecGbw1MXnaALl5ZlA+BxlwORG22PQrNzAt7jIdKQlzktGi+luZMw0RFAi76So+aWEvhO3mcLGpk7LxIvF58bTnvMRUN0WWhhRP+ltnRMH09mDs5nTN7HpEz6KTafaDtfnjF/jf77ezpnURm8HrFbscCNW2xxLcrTQZE+0Gneo8BwLcYcST7MGd+ix++W3iT905Z0TJWi6wXkpXsgqJyKpClmVJReKZKdM8LKM/JHoYaC7qskiKp8ZODviR+x0mOcIn2i51Z4ygpq0eLYgatLjCeOoSZUnCqUzFaDRVAmcOr5CPuvurVXtiEEO6xwhCC022lNGIIMSzxaADGo8YfwxaPJE4WeqgY1Jx/amlM41HOlInZVrYx3pSLdtjmCGFprs2UKZWYUnCmRmJZ4rjJn1eI4gpuF/pCNnBnrGdOSI+BE7HeY5widabrWnjKA/QDrSqscTxtHnTkfqVfi505GNTY50JMg6RwhCi432lBHo+dORNjWeMP48dTpSq4HrmGUpeEqk5OHuQjU2q3kU8sJFjhXlhU635Zdle84JTNZ4v7oRNaCw/lB9iYuNmIFE9byUIHhaWgsFSVY7bs+vux04ULfRwH5Xf+N6VPbbmzDlvQ0wLrjj5LhJLRETSaPwwWK5kDit5+5Gbw1UYr+24Hzq+5f71HgV6WqaP3ZJuruGP2FNurHmeytKp2e3l6p0Wna7KkunZbizunRajvsrTGegua/KdFqSeypNpyW4o9p0Wn57Kk6nJbiv6nRairsqT6dnuLv6dFqauyxQZ2K6rwp1WpY/W4m6sRF2V6NOR29fRep0DHdUpU5Hb9MydQ0Bc506Q2WI5ftehalumbh+u5l9FDinaaMXTcmpwYpTkay/+l4rk3gIHpXBiL8h3eJAzdgWrWIJ9PjMBVNGstM3QZXxO2lEmjDx/RFfgrNTVTRIn2FUoPzJZ4DPYOoWPXIvn0ERakWpuPgUfma5Q2rYuAAO50WYzE+S/hl3sTc2wl21xCl6sOKryInx62cxdRDQJLml5ZDh8nD+YOeS1E+jaacWtNO8drdg+ZEz272OP2Fue6r73rLbJn57yW8b+O0qw23guLMct4Hl/rLcRqL7ynMbaO4p022guKNct4HhnrLdBor7yncbSO4q423iuLuct4HoLrPeZq77ynsbeP5sme+pGXaX+9YT3Ff2W89xR/lvPcFNM+D3/4yBO3HntifrWIpPF9Mbl/pDYzqqM5q0uPBkBtHPIpxP3YEv9gyij3Mf5VrsGUQ/htXMSflizyB6D0iEpHy6kfCFn6KY78f6zwj5IRUJ19o2Lask5frGMp4vRcssMSM+ilLgLPk1mEAD3Eja17DS5nBj3QKL0+CN5BVcxDmR/fLrS7hE1VCjmrF2Cgxlf40qXDdGA+lRhX/VvtqMnicBA616t0taEkaLSFlMRp1fIb2MdNc+6VgnNgmjjXOC07eQtwZmTFxSRh9mN1g/K/oeYRMxU/v+MckhL/rnYhMvv8/E2/FyyOtpUaZIlK+DRmIuib3jqM6MytuW3NwiH99mVIy8qImYXdwjr6NiZJVNpPTi+k1XpUptLmq919/o6InNoaP15mwN9PBvNmFbed0pJ5DPPWOV3mKcfdGzconbLBSMaUEDQVaVjKY4SurJwKyT6LZZ7Cg1JgaMUYrzHBcRve2YlUvcRt52Ssok7BGRcJacP8yjvGINeufiEDFiAvimv5qKRYYlBRSOhBG8k96cyos8KC4ZYy6Rh8Uq5JHWLi5U16dt/cog51nFSCKqonC1glukZkvdij+hIhUWMTaodyYW+BkRHvIo8YzHFN2QfUarkglNYpOUN5ITgdlJ+53IM8XlhhwJr0pFc5LkAVKVDrw++46LjFlSo6sq89KcquSGRcDExZ1v4sAeU5D8EvA02piCEbvfqpTGUzsrhGtRH0n4vPnrKKN6TU8F+R7Qnh3bxIY8UI2LjxNmjOu6262l0MInZvgJD9syLQAPwMJMBnS7vXgdah9gTZMnQMK1rKLvu2OUIGlN0nwWAlQZSS8yKYmgPEuqCPvaARmYpCGvPyqu8Ba0nIIeh/j1jmxMJ2Q9vNZiNpkIbAbkMEX7DxCJ/Z7jJrhSjGSgp8Ly9JM9Z6wtVBOfrEHswOvgNFXmV3eBJ32wsUFPGTTu3myl9TysAsahLYY57MgD+fe1VeC97ZgFRMyQUiXxNfh2e07JJsYeJ9fH3/tHXEXTt5A61q+kC3lGwbQNCkHCjD1Kv7V3lsJycAGPCUQYCRfwlECEcXBD9+egJBExjODAnYiPYIIR8pep3NnXLf8zVIRhqWgqCRbpLViG5Fyxt9P9flmc3aXrqPECe3e57qFFkpEmiV3qI7WYl4wErd4I4GaW+kgdZeQ70SZeAwyI7TNdKI07DWwfBftREMTyfSs4I5u4x901qQT/2I7UWOAwGSpP9fOdBefDPx3yXRzjJUnzv1GO8d6xoxzjvWNHOcZ7x451jPcObzrGW4cmqXBezubEFLvFffmX/omXGdhwRqJFk0s/hAFngXH2LomS9btKFUlVJcwHE+2J/JpPYsUZMtdYxkuQEcPyAnoJsILzOIGrZFhd9KeG3dyRbgo1qnCZWJHt72cA6Trgx0qgDhq6GzYeb4oJZCyk/v8plLeQOcjku+JKVYwoEzErlTGijBf0Kz2YCaR3s215pWBubP33T5LcOelSPgF8i0aQ1dOY38mF0oaAmpXU3PYZkamgpaK8eEV/cazJRkZrsd7Ixzc++uapRWx//kqwQn/712Qe3ccLlqXQ9XNz8PGSNWWVVEQkAF38lgsuclMUQRjBsgbCwEXHhu/T3PbjR63viOth16zXPT/y9cEsYC95w7jm0MvQAuZ1gk2FoEdENOtEO4upDosfNhwHgz2s2YjAH14zf7rX37gTsj0cdFGe8kJhWhDTlXuYLSKtVd+IKIj9HIGJGXJPMD8Q0LoZAgRZAduGP8cFvpKcFCohBT7PD/O2EGfOGcFTMZbwUP/8TaKUFEpgNhCD7mL+edmKlBaKXGeFwhxE/rPKz0QgfrnjS6TTFbxctdnTtQjVj6mLP85Jzb4FR5UkGTp/NCFYS6K9ZxaHQ4ONzoQWVz0R05mC5SsMuWqF8bhjH8zN4TTONVlIH0NLGB9DWZschjvqA6twYeQ7hHJ2HQx/DMVZUA6oAvKwyAJIZ922BZiQUmsLYAGl0RahgmqZLUB2Fh5bgAmoFbYA1VXcawEktCGkJyysQaMvqLvk1QJUj96Q/siAglILgAHln4CoI8RN/S6gdhNQBxTX88JKJC0AtlY0AuA9ppm+EpEDoh+IKylUeolzl8d2PgImxSVpKC3F6Y0k9YaDEnfDXo/XowUW5CKIvDVXHit9oZvV2LJKm5LlgcAvRKW3JCzaBTN2xqm7HpcHJrRtpQekTzk/6wvyO9bsN1GQ1+MqSvv1ijDrc8ZI6n4TQnp3aCt4tGDF4m7eDtcJRe9N7tQL+ex/xtgQ7ZCHhmiBlshPU7Rg7blSDLyD+Qohnh3GPSWNlmnATvReLtPQXmeOFtJJNNfiE0FSQt+JT09zD19xuKHFbijkPJtKCDaR+8W0YnEuy20Sohm/bpv1Ol6MIz47RRzxeTfxeV7vVQ8WNDPOC1nlbUXiqiy5UCSz9Ary1Awd3kjD4PBGRhGHN9qNN1ICp4Cc0rFgsf8cLsLO5nARz+Ui+gNlSpkm0bFl0kMfHujwQB5iDg90bJkOb3R4I4eIwxvtxhsdWybrz+EidFQOF/Eju4j+JojgKZGSxzli0kzntgv7hYscN43Yrbsz5/6uXVmFxZQlLgIjKkGw/XrNpj0o93oA3n30CqgDCn5UdxA8C4nTelKsnyO95+M8TmeT4w7HcYdDC3nc4TjucBx3OI47HI9l5HGHQ4N63OEAQwCOb65fzyhM7XfmQyxAso8C5zQ93auBnBrMQNcWrK3mYGpAVBmpEyPn27baOjX9eWAlvsHQN4KzU1U08yg07fv+NjRsFK5ScXsjTSDmEA9bo134ucdkfpL0z/DxqiInxq8BsDX7Oun4TAJxgror/ujY0hl+ji1dAMhjS3ds6Y4t3bGlO7Z0VtRjSweCeDjVC/+ie2JJqaJiTTm8VYX41FPW0gtayPAwPuz5fg3f9I4JZv6mDRrXl2oA7CgYx2YzrNlPvPz6YvUMGa9cXTJefv26HmM9ROEqYRhiK2M2F4grgpnNH2sZ1GByBpvojsZPkN2zq2sTAMPedQkMsEILV88jAIS1aRH8eU8GUUI/qMmyMxY4myUbldH2MF3lzaN4Y0j/TuCiD9BzE4jk6AsIqQxr6+kZzjm7e1T6HJP0aH25CNbaGxPwdltbTcJeKECXSACRtpdhlPchwNTzaS4JOG8Gbwtp9EWGZt3LvZG5jTZ49IwNsOEIptbVkANQ2p7WDjPqG9kvt6KhbTtkcW5rTQ9+3tBX3mEFU0w0inWUVm4KHbegWnmiKhqBVy54pWhhGAB/wY+62A/oWlpBiyuihbnANKNnr42yg8e/N9MEpTzPeYEUR5ixRvhUUdd8KmlJGC3M689VCwVGLV+Sgd7Mli0OF5Vx+rb2TCq6H/Q9tX/b5PC039e6H8cYD38e8w6nbwX/xkh2JZnrS32vXJaRzBoMdqViQ7QlXU9tdSOOTwZjPYHHF3agaMt0iab3i3NPN6gdb3QRPF+i79MN7lKNL5SxKON7xZX129OK0W0513HQW88I4xpF0fuortC0BJQWvTA+Wytsoed/E5GSQuFr07iiV+1O3alhjr+f9j2Q/4G/07zKUTHzwYzxb74jKkjO35/PCd9p+3qkTtunc8Ee+oKOQIZcdVq6WCO/nI37CKUHWHt0MhAWZYrYTwl6oJXVmVF5CwUniLIexPFJmhlSCwCowTLnQuOcSr5vUtftxaTCYu1+TipuPKgMPhPIODbmOgF2bttCedsZ4IT+q204ZUolg86P679EI1BCEewp//5RDjpkxXSGQEL/VkvpIoFJ2qbb+VV7usE2fZMwsDidUIuGJhLOWKW39Uud5YZ1MggaDddNAReFnmpVMpriAEvIVWw7Fm7CIRYHK7dfdgaDk5Y5LlaG94VMFeeoFg59tVasHhYx/Xstb7YPcoRd7fEeFDIK/A/BmfVDLDQCBNmTrdpkGY3lfVEgBsuan01+R7G5PRV71P+3FvJDDHttLmW5irCDkW9G1EWg/26FFSlStwsNeab6RqXiV2EsRuInGSodeZ6/RvCj0micIwMDu9KAE2SS0VnD4UCsy9++RmJd/vbbCmTrhm7FQTBHG7b12/Fzxd5O92tyG+SZ1nX2msKlPC8ZWVXE53F+LyPfzfu9fabaUkGsC2W/vB2RSvAPKNw/AgAA//+OdEJI" } diff --git a/metricbeat/module/beat/stats/_meta/fields.yml b/metricbeat/module/beat/stats/_meta/fields.yml index bb48a6c7b651..a1f670a155ac 100644 --- a/metricbeat/module/beat/stats/_meta/fields.yml +++ b/metricbeat/module/beat/stats/_meta/fields.yml @@ -358,8 +358,38 @@ type: long - name: unset type: long - - name: sampling.transactions_dropped - type: long + - name: sampling + type: group + fields: + - name: tail + type: group + fields: + - name: dynamic_service_groups + type: long + - name: events + type: group + fields: + - name: dropped + type: long + - name: failed_writes + type: long + - name: head_unsampled + type: long + - name: processed + type: long + - name: sampled + type: long + - name: stored + type: long + - name: storage + type: group + fields: + - name: lsm_size + type: long + - name: value_log_size + type: long + - name: transactions_dropped + type: long - name: server type: group fields: diff --git a/metricbeat/module/beat/stats/_meta/test/apm-server.stats.712.json b/metricbeat/module/beat/stats/_meta/test/apm-server.stats.712.json index e1f81cd2ef64..e25bd55b5f28 100644 --- a/metricbeat/module/beat/stats/_meta/test/apm-server.stats.712.json +++ b/metricbeat/module/beat/stats/_meta/test/apm-server.stats.712.json @@ -36,7 +36,7 @@ "elasticsearch": { "cache.entries.count": 0, "cache.refresh.failures": 0, - "cache.refresh.successes": 220, + "cache.refresh.successes": 6, "fetch.es": 0, "fetch.fallback": 0, "fetch.invalid": 0, @@ -199,23 +199,23 @@ }, "processor": { "error": { - "transformations": 2563 + "transformations": 782 }, "metric": { - "transformations": 13792 + "transformations": 8295 }, "span": { - "transformations": 66999 + "transformations": 10234 }, "stream": { - "accepted": 111954, + "accepted": 41772, "errors": { "invalid": 0, "toolarge": 0 } }, "transaction": { - "transformations": 29994 + "transformations": 3004 } }, "root": { @@ -226,7 +226,7 @@ "count": 1, "errors": { "closed": 0, - "count": 0, + "count": 1, "decode": 0, "forbidden": 0, "internal": 0, @@ -235,7 +235,7 @@ "notfound": 0, "queue": 0, "ratelimit": 0, - "timeout": 0, + "timeout": 1, "toolarge": 0, "unauthorized": 0, "unavailable": 0, @@ -243,28 +243,43 @@ }, "valid": { "accepted": 0, - "count": 1, + "count": 0, "notmodified": 0, - "ok": 1 + "ok": 0 } }, "unset": 0 }, "sampling": { + "tail": { + "dynamic_service_groups": 4, + "events": { + "dropped": 2149, + "failed_writes": 0, + "head_unsampled": 0, + "processed": 34538, + "sampled": 13238, + "stored": 32389 + }, + "storage": { + "lsm_size": 525060325, + "value_log_size": 12058435 + } + }, "transactions_dropped": 0 }, "server": { "request": { - "count": 1492 + "count": 605 }, "response": { - "count": 1492, + "count": 605, "errors": { "closed": 0, - "count": 1, + "count": 0, "decode": 0, "forbidden": 0, - "internal": 1, + "internal": 0, "invalidquery": 0, "method": 0, "notfound": 0, @@ -277,8 +292,8 @@ "validate": 0 }, "valid": { - "accepted": 1491, - "count": 1491, + "accepted": 605, + "count": 605, "notmodified": 0, "ok": 0 } @@ -302,29 +317,29 @@ "id": "session-c2.scope", "mem": { "usage": { - "bytes": 37939621888 + "bytes": 30244999168 } } } }, "cpu": { "system": { - "ticks": 1720, + "ticks": 2360, "time": { - "ms": 1720 + "ms": 2360 } }, "total": { - "ticks": 16100, + "ticks": 12850, "time": { - "ms": 16100 + "ms": 12850 }, - "value": 16100 + "value": 12850 }, "user": { - "ticks": 14380, + "ticks": 10490, "time": { - "ms": 14380 + "ms": 10490 } } }, @@ -333,53 +348,53 @@ "hard": 1048576, "soft": 1048576 }, - "open": 18 + "open": 56 }, "info": { - "ephemeral_id": "87279482-5cbc-4232-b90e-a08d2d2f0ec9", + "ephemeral_id": "d57303bc-7918-4832-a917-e8f2467ca5fd", "name": "apm-server", "uptime": { - "ms": 6589405 + "ms": 179556 }, - "version": "8.14.3" + "version": "8.18.0" }, "memstats": { - "gc_next": 204429288, - "memory_alloc": 102433408, - "memory_sys": 267778360, - "memory_total": 1392024448, - "rss": 164769792 + "gc_next": 178933328, + "memory_alloc": 129688560, + "memory_sys": 226933048, + "memory_total": 1972011464, + "rss": 762793984 }, "runtime": { - "goroutines": 52 + "goroutines": 33 } }, "libbeat": { "output": { "events": { - "acked": 70666, - "active": 0, - "batches": 26, - "failed": 42682, - "toomany": 42682, - "total": 113348 + "acked": 18441, + "active": 3874, + "batches": 7, + "failed": 0, + "toomany": 0, + "total": 22315 }, "type": "elasticsearch", "write": { - "bytes": 24728235 + "bytes": 3220280 } }, "pipeline": { "events": { - "total": 113348 + "total": 22315 } } }, "output": { "elasticsearch": { "bulk_requests": { - "available": 60, - "completed": 26 + "available": 37, + "completed": 7 }, "indexers": { "active": 1, @@ -393,13 +408,13 @@ "cores": 16 }, "load": { - "1": 2.88, - "15": 1.86, - "5": 1.87, + "1": 14.04, + "15": 5.22, + "5": 8.9, "norm": { - "1": 0.18, - "15": 0.1163, - "5": 0.1169 + "1": 0.8775, + "15": 0.3263, + "5": 0.5563 } } }