From 91d2e6728de0f6ab0848add22f931b3e8f12ab06 Mon Sep 17 00:00:00 2001 From: Jakub Surdej Date: Wed, 9 Oct 2024 14:33:57 +0200 Subject: [PATCH] Update snapshots and fixtures to match Logstash 8.15 --- Makefile | 4 + README.md | 1 + fixtures/node_info.json | 16 +- fixtures/node_stats.json | 499 +++++++++++------- .../__snapshots__/nodeinfo_response_test.snap | 16 +- .../nodestats_response_test.snap | 180 ++++--- 6 files changed, 439 insertions(+), 277 deletions(-) diff --git a/Makefile b/Makefile index 79191b20..7a358cf9 100644 --- a/Makefile +++ b/Makefile @@ -138,6 +138,10 @@ migrate-v1-to-v2: update-readme-descriptions: ./scripts/add_descriptions_to_readme.sh +#: Updates snapshot for test data and runs tests +update-snapshots: + UPDATE_SNAPS=true go test ./... + #: Shows info about available commands help: @grep -B1 -E "^[a-zA-Z0-9_-]+:([^\=]|$$)" Makefile \ diff --git a/README.md b/README.md index 070a60a0..a060e1f4 100644 --- a/README.md +++ b/README.md @@ -181,6 +181,7 @@ See more in the [Migration](#migration) section. - `make upgrade-dependencies`: Upgrades all dependencies. - `make migrate-v1-to-v2`: Migrates configuration from v1 to v2. - `make update-readme-descriptions`: Update Makefile descriptions in main README.md. +- `make update-snapshots`: Updates snapshot for test data and runs tests. - `make help`: Shows info about available commands. diff --git a/fixtures/node_info.json b/fixtures/node_info.json index c7964437..023dad98 100644 --- a/fixtures/node_info.json +++ b/fixtures/node_info.json @@ -1,18 +1,18 @@ { - "host": "9e6e14cf5532", - "version": "8.6.1", + "host": "814a8393fbd5", + "version": "8.15.2", "http_address": "0.0.0.0:9600", - "id": "a2c7110e-5ccf-4226-bc9b-e773710e66a0", - "name": "9e6e14cf5532", - "ephemeral_id": "4a2ee04f-2733-4eaa-887d-675bad27f07c", + "id": "690de5cc-deb1-48d9-ba02-d4ec1b22e62a", + "name": "814a8393fbd5", + "ephemeral_id": "eb4d9042-5642-4e21-bb8d-27454b81c5bc", "status": "green", "snapshot": false, "pipeline": { - "workers": 16, + "workers": 10, "batch_size": 125, "batch_delay": 50 }, - "build_date": "2023-01-24T10:41:55+00:00", - "build_sha": "6a248b5091c490f09460db5651e5239d903b97cf", + "build_date": "2024-07-24T09:37:48+00:00", + "build_sha": "46b996c24da17cdc7a16bc3037edab5c6132ccd0", "build_snapshot": false } diff --git a/fixtures/node_stats.json b/fixtures/node_stats.json index 44059ded..e269266a 100644 --- a/fixtures/node_stats.json +++ b/fixtures/node_stats.json @@ -1,58 +1,58 @@ { - "host": "f26e584ecf05", - "version": "8.8.1", + "host": "814a8393fbd5", + "version": "8.15.0", "http_address": "0.0.0.0:9600", - "id": "096d672d-50d5-420b-a27c-254c089bdd78", - "name": "f26e584ecf05", - "ephemeral_id": "25135ee3-be69-4076-bda1-e27524d9ee93", + "id": "690de5cc-deb1-48d9-ba02-d4ec1b22e62a", + "name": "814a8393fbd5", + "ephemeral_id": "eb4d9042-5642-4e21-bb8d-27454b81c5bc", "status": "green", "snapshot": false, "pipeline": { - "workers": 16, + "workers": 10, "batch_size": 125, "batch_delay": 50 }, "jvm": { "threads": { - "count": 60, - "peak_count": 60 + "count": 65, + "peak_count": 65 }, "mem": { - "heap_used_percent": 27, + "heap_used_percent": 36, "heap_committed_in_bytes": 1073741824, "heap_max_in_bytes": 1073741822, - "heap_used_in_bytes": 294044784, - "non_heap_used_in_bytes": 147703688, - "non_heap_committed_in_bytes": 155189248, + "heap_used_in_bytes": 395837440, + "non_heap_used_in_bytes": 172607568, + "non_heap_committed_in_bytes": 183304192, "pools": { "young": { - "peak_max_in_bytes": -1, + "committed_in_bytes": 668991488, + "peak_used_in_bytes": 351272960, "max_in_bytes": -1, - "committed_in_bytes": 346030080, - "peak_used_in_bytes": 326107136, - "used_in_bytes": 180355072 - }, - "old": { - "peak_max_in_bytes": 1073741824, - "max_in_bytes": 1073741824, - "committed_in_bytes": 687865856, - "peak_used_in_bytes": 73986560, - "used_in_bytes": 73986560 + "used_in_bytes": 257949696, + "peak_max_in_bytes": -1 }, "survivor": { - "peak_max_in_bytes": -1, + "committed_in_bytes": 7340032, + "peak_used_in_bytes": 47185920, "max_in_bytes": -1, - "committed_in_bytes": 39845888, - "peak_used_in_bytes": 39703152, - "used_in_bytes": 39703152 + "used_in_bytes": 7340032, + "peak_max_in_bytes": -1 + }, + "old": { + "committed_in_bytes": 397410304, + "peak_used_in_bytes": 132644864, + "max_in_bytes": 1073741824, + "used_in_bytes": 130547712, + "peak_max_in_bytes": 1073741824 } } }, "gc": { "collectors": { "young": { - "collection_count": 8, - "collection_time_in_millis": 224 + "collection_count": 11, + "collection_time_in_millis": 110 }, "old": { "collection_count": 0, @@ -60,188 +60,379 @@ } } }, - "uptime_in_millis": 53120 + "uptime_in_millis": 56226 }, "process": { - "open_file_descriptors": 98, - "peak_open_file_descriptors": 98, + "open_file_descriptors": 108, + "peak_open_file_descriptors": 109, "max_file_descriptors": 1048576, "mem": { - "total_virtual_in_bytes": 9305346048 + "total_virtual_in_bytes": 8282685440 }, "cpu": { - "total_in_millis": 135300, + "total_in_millis": 35730, "percent": 0, "load_average": { - "1m": 3.79, - "5m": 1.29, - "15m": 0.46 + "1m": 2.25, + "5m": 0.68, + "15m": 0.24 } } }, "events": { - "in": 4001, - "filtered": 10, - "out": 2, - "duration_in_millis": 5, - "queue_push_duration_in_millis": 7 + "in": 3751, + "filtered": 1250, + "out": 1250, + "duration_in_millis": 494960, + "queue_push_duration_in_millis": 49451 }, "flow": { "input_throughput": { - "current": 1.0, - "lifetime": 117.4 + "current": 0.0, + "lifetime": 73.9 }, "filter_throughput": { - "current": 2.1, - "lifetime": 3.2 + "current": 0.0, + "lifetime": 24.63 }, "output_throughput": { - "current": 4.3, - "lifetime": 5.4 + "current": 0.0, + "lifetime": 24.63 }, "queue_backpressure": { - "current": 6.5, - "lifetime": 7.6 + "current": 1.0, + "lifetime": 0.9743 }, "worker_concurrency": { - "current": 8.7, - "lifetime": 9.8 + "current": 10.0, + "lifetime": 9.752 } }, "pipelines": { - "main": { + ".monitoring-logstash": { "events": { - "out": 0, "filtered": 0, - "in": 4001, "duration_in_millis": 0, - "queue_push_duration_in_millis": 0 + "queue_push_duration_in_millis": 0, + "out": 0, + "in": 0 }, "flow": { - "output_throughput": { - "current": 2.3, - "lifetime": 3.4 + "input_throughput": { + "current": 0.0, + "lifetime": 0.0 + }, + "filter_throughput": { + "current": 0.0, + "lifetime": 0.0 }, "worker_concurrency": { - "current": 4.5, - "lifetime": 5.6 + "current": 0.0, + "lifetime": 0.0 + }, + "queue_backpressure": { + "current": 0.0, + "lifetime": 0.0 }, + "output_throughput": { + "current": 0.0, + "lifetime": 0.0 + }, + "worker_utilization": { + "current": 0.0, + "lifetime": 0.0 + } + }, + "plugins": { + "inputs": [ + { + "id": "9a9bed30135e19c8047fe6aa0588b70b15280fb9161fea8ed8e7368e1fb1e6d3", + "events": { + "out": 0 + }, + "flow": { + "throughput": { + "current": 0.0, + "lifetime": 0.0 + } + } + } + ], + "codecs": [], + "filters": [], + "outputs": [ + { + "id": "e7aab1e17ec42d6573f3be4b8fce17c5dc69db8473f9505f386b1160434b141b", + "events": { + "duration_in_millis": 0, + "in": 0 + }, + "flow": { + "worker_millis_per_event": {}, + "worker_utilization": { + "current": 0.0, + "lifetime": 0.0 + } + } + } + ] + }, + "reloads": { + "last_failure_timestamp": "2023-04-20T20:00:32.437218256Z", + "successes": 3, + "failures": 1, + "last_success_timestamp": "2023-04-20T22:30:32.437218256Z", + "last_error": { + "message": "No configuration found in the configured sources.", + "backtrace": [ + "org/logstash/execution/AbstractPipelineExt.java:151:in `reload_pipeline'", + "/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:181:in `block in reload_pipeline'", + "/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/stud-0.0.23/lib/stud/task.rb:24:in `block in initialize'" + ] + } + }, + "queue": null + }, + "main": { + "events": { + "filtered": 1250, + "duration_in_millis": 495018, + "queue_push_duration_in_millis": 49455, + "out": 1250, + "in": 3751 + }, + "flow": { "input_throughput": { - "current": 6.7, - "lifetime": 124.0 + "current": 0.0, + "lifetime": 74.88 }, "filter_throughput": { - "current": 7.8, - "lifetime": 8.9 + "current": 0.0, + "lifetime": 24.95 + }, + "worker_concurrency": { + "current": 10.0, + "lifetime": 9.882 }, "queue_backpressure": { - "current": 1.1, - "lifetime": 2.2 + "current": 1.0, + "lifetime": 0.9872 + }, + "output_throughput": { + "current": 0.0, + "lifetime": 24.95 + }, + "worker_utilization": { + "current": 100.0, + "lifetime": 98.82 } }, "plugins": { "inputs": [ { - "id": "c75c0c6f97fd2c8605b95a5b2694fdae97189fe49553787a923faeaa3342c54a", + "id": "5ee0ea3d45c32bab3b41963bd900e758ba6e193a11079649302574c706fd5e2f", + "name": "dead_letter_queue", + "events": { + "queue_push_duration_in_millis": 0, + "out": 0 + }, + "flow": { + "throughput": { + "current": 0.0, + "lifetime": 0.0 + } + } + }, + { + "id": "95bb3e4f2a40f87147b6ab5035e08ba31858eace7604a57d2e719db790097222", "name": "generator", "events": { - "out": 4001, - "queue_push_duration_in_millis": 0 + "queue_push_duration_in_millis": 49454, + "out": 3751 + }, + "flow": { + "throughput": { + "current": 0.0, + "lifetime": 74.88 + } } } ], "codecs": [ { - "id": "plain_d7fb272d-75f5-4e38-bf56-c4c798bea2d1", - "name": "plain", + "id": "plain_4ffdd0bf-b707-419b-b425-d3986a43c35a", "decode": { - "out": 0, "writes_in": 0, - "duration_in_millis": 0 + "duration_in_millis": 0, + "out": 0 }, + "name": "plain", "encode": { "writes_in": 0, "duration_in_millis": 0 } }, { - "id": "plain_35cb2d25-0d8d-441a-a714-82121715864d", - "name": "plain", + "id": "plain_b4b163b6-effd-454a-9605-c3f8ef0cde5e", "decode": { - "out": 4001, - "writes_in": 4001, - "duration_in_millis": 57 + "writes_in": 0, + "duration_in_millis": 0, + "out": 0 }, + "name": "plain", "encode": { "writes_in": 0, "duration_in_millis": 0 } + }, + { + "id": "plain-codec-001", + "name": "plain" } ], "filters": [ { - "id": "1721e7c39758977b227be1d9334f0752555f39c873b8b86a3df8546f64059112", - "name": "json", + "id": "prune-http-input-fields", + "name": "prune", "events": { - "out": 2000, - "in": 2000, - "duration_in_millis": 716 + "duration_in_millis": 127, + "out": 1250, + "in": 1250 + }, + "flow": { + "worker_millis_per_event": { + "lifetime": 0.1016 + }, + "worker_utilization": { + "current": 0.0, + "lifetime": 0.02535 + } } }, { - "id": "drop_b8ed8ea8c0ace91d6b617f6c8a5153141183c35a330de014182825dbceeade00", + "id": "ca953dac49c8fd3b00ba8275af10f9c6bcd9ca95755cd7892952966c5a13d427", + "name": "ruby", + "events": { + "duration_in_millis": 489610, + "out": 1250, + "in": 2500 + }, + "flow": { + "worker_millis_per_event": { + "current": "Infinity", + "lifetime": 195.8 + }, + "worker_utilization": { + "current": 100.0, + "lifetime": 97.74 + } + } + }, + { + "id": "drop-non-existent", "name": "drop", "events": { - "out": 300, - "in": 330, - "duration_in_millis": 333 + "duration_in_millis": 0, + "out": 0, + "in": 0 + }, + "flow": { + "worker_millis_per_event": {}, + "worker_utilization": { + "current": 0.0, + "lifetime": 0.0 + } + } + }, + { + "id": "json-filter", + "name": "json", + "events": { + "duration_in_millis": 214, + "out": 1250, + "in": 1250 + }, + "flow": { + "worker_millis_per_event": { + "lifetime": 0.1712 + }, + "worker_utilization": { + "current": 0.0, + "lifetime": 0.04272 + } + } + }, + { + "id": "mutate-path-001", + "name": "mutate", + "events": { + "duration_in_millis": 170, + "out": 1250, + "in": 1250 + }, + "flow": { + "worker_millis_per_event": { + "lifetime": 0.136 + }, + "worker_utilization": { + "current": 0.0, + "lifetime": 0.03394 + } } }, { - "id": "drop_e2e0f559b7292f788693f9f318185d5c1d30127870ca8f0e608b11d9dc560079", + "id": "drop-80-percent", "name": "drop", "events": { - "out": 800, - "in": 880, - "duration_in_millis": 888 + "duration_in_millis": 0, + "out": 0, + "in": 0 + }, + "flow": { + "worker_millis_per_event": {}, + "worker_utilization": { + "current": 0.0, + "lifetime": 0.0 + } } } ], "outputs": [ { - "id": "45554a51a53a57f5dbba7d26b65aad526147453a895529f3d4698c8fd88692ef", + "id": "81d12b78dc90935689721a2220cbd7ad2f75910cdafdc01b5b77f6ca8f356cef", "name": "elasticsearch", "events": { - "out": 0, - "in": 2000, - "duration_in_millis": 0 - }, - "documents": { - "successes": 1337, - "non_retryable_failures": 87 + "duration_in_millis": 4887, + "out": 1250, + "in": 1250 }, "bulk_requests": { - "with_errors": 87, + "successes": 10, "responses": { - "200": 87 + "200": 10 + } + }, + "documents": { + "successes": 1250 + }, + "flow": { + "worker_millis_per_event": { + "lifetime": 3.91 + }, + "worker_utilization": { + "current": 0.0, + "lifetime": 0.9756 } } } ] }, "reloads": { - "last_failure_timestamp": "2023-04-20T20:00:32.437218256Z", - "successes": 3, - "failures": 1, - "last_success_timestamp": "2023-04-20T22:30:32.437218256Z", - "last_error": { - "message": "No configuration found in the configured sources.", - "backtrace": [ - "org/logstash/execution/AbstractPipelineExt.java:151:in `reload_pipeline'", - "/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:181:in `block in reload_pipeline'", - "/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/stud-0.0.23/lib/stud/task.rb:24:in `block in initialize'" - ] - } + "last_success_timestamp": null, + "failures": 0, + "last_error": null, + "successes": 0, + "last_failure_timestamp": null }, "queue": { "type": "memory", @@ -250,84 +441,22 @@ "max_queue_size_in_bytes": 0 }, "dead_letter_queue": { - "max_queue_size_in_bytes": 47244640256, - "last_error": "no errors", "queue_size_in_bytes": 1, - "dropped_events": 0, + "last_error": "no errors", "expired_events": 0, - "storage_policy": "drop_newer" - }, - "hash": "a73729cc9c29203931db21553c5edba063820a7e40d16cb5053be75cc3811a17", - "ephemeral_id": "a5c63d09-1ba6-4d67-90a5-075f468a7ab0" - }, - ".monitoring-logstash": { - "events": { - "out": 0, - "filtered": 0, - "in": 0, - "duration_in_millis": 0, - "queue_push_duration_in_millis": 0 - }, - "flow": { - "output_throughput": { - "current": 0.0, - "lifetime": 0.0 - }, - "worker_concurrency": { - "current": 0.0, - "lifetime": 0.0 - }, - "input_throughput": { - "current": 0.0, - "lifetime": 0.0 - }, - "filter_throughput": { - "current": 0.0, - "lifetime": 0.0 - }, - "queue_backpressure": { - "current": 0.0, - "lifetime": 0.0 - } - }, - "plugins": { - "inputs": [], - "codecs": [], - "filters": [], - "outputs": [] - }, - "reloads": { - "last_failure_timestamp": null, - "successes": 0, - "failures": 0, - "last_success_timestamp": null, - "last_error": null + "storage_policy": "drop_newer", + "dropped_events": 0, + "max_queue_size_in_bytes": 1073741824 }, - "queue": null + "hash": "d30c4ff4da9fdb1a6b06ee390df1336aa80cc5ce6582d316af3dc0695af2d82e", + "ephemeral_id": "31caf4d6-162d-4eeb-bc04-411ae2e996f1" } }, "reloads": { - "successes": 0, - "failures": 0 - }, - "os": { - "cgroup": { - "cpu": { - "cfs_period_micros": 100000, - "cfs_quota_micros": -1, - "stat": { - "time_throttled_nanos": 0, - "number_of_times_throttled": 0, - "number_of_elapsed_periods": 0 - }, - "control_group": "/" - }, - "cpuacct": { - "usage_nanos": 161531487900, - "control_group": "/" - } - } + "failures": 0, + "successes": 0 }, + "os": {}, "queue": { "events_count": 0 } diff --git a/internal/fetcher/responses/__snapshots__/nodeinfo_response_test.snap b/internal/fetcher/responses/__snapshots__/nodeinfo_response_test.snap index 0929d163..51493285 100755 --- a/internal/fetcher/responses/__snapshots__/nodeinfo_response_test.snap +++ b/internal/fetcher/responses/__snapshots__/nodeinfo_response_test.snap @@ -2,17 +2,17 @@ [TestNodeInfoResponseStructure - 1] Unmarshalled NodeInfoResponse responses.NodeInfoResponse{ - Host: "9e6e14cf5532", - Version: "8.6.1", + Host: "814a8393fbd5", + Version: "8.15.2", HTTPAddress: "0.0.0.0:9600", - ID: "a2c7110e-5ccf-4226-bc9b-e773710e66a0", - Name: "9e6e14cf5532", - EphemeralID: "4a2ee04f-2733-4eaa-887d-675bad27f07c", + ID: "690de5cc-deb1-48d9-ba02-d4ec1b22e62a", + Name: "814a8393fbd5", + EphemeralID: "eb4d9042-5642-4e21-bb8d-27454b81c5bc", Status: "green", Snapshot: false, - Pipeline: struct { Workers int "json:\"workers\""; BatchSize int "json:\"batch_size\""; BatchDelay int "json:\"batch_delay\"" }{Workers:16, BatchSize:125, BatchDelay:50}, - BuildDate: "2023-01-24T10:41:55+00:00", - BuildSHA: "6a248b5091c490f09460db5651e5239d903b97cf", + Pipeline: struct { Workers int "json:\"workers\""; BatchSize int "json:\"batch_size\""; BatchDelay int "json:\"batch_delay\"" }{Workers:10, BatchSize:125, BatchDelay:50}, + BuildDate: "2024-07-24T09:37:48+00:00", + BuildSHA: "46b996c24da17cdc7a16bc3037edab5c6132ccd0", BuildSnapshot: false, } --- diff --git a/internal/fetcher/responses/__snapshots__/nodestats_response_test.snap b/internal/fetcher/responses/__snapshots__/nodestats_response_test.snap index b13a0a7d..44e708f5 100755 --- a/internal/fetcher/responses/__snapshots__/nodestats_response_test.snap +++ b/internal/fetcher/responses/__snapshots__/nodestats_response_test.snap @@ -2,69 +2,59 @@ [TestNodeStatsResponseStructure - 1] Unmarshalled NodestatsResponse responses.NodeStatsResponse{ - Host: "f26e584ecf05", - Version: "8.8.1", + Host: "814a8393fbd5", + Version: "8.15.0", HttpAddress: "0.0.0.0:9600", - Id: "096d672d-50d5-420b-a27c-254c089bdd78", - Name: "f26e584ecf05", - EphemeralId: "25135ee3-be69-4076-bda1-e27524d9ee93", + Id: "690de5cc-deb1-48d9-ba02-d4ec1b22e62a", + Name: "814a8393fbd5", + EphemeralId: "eb4d9042-5642-4e21-bb8d-27454b81c5bc", Status: "green", Snapshot: false, - Pipeline: responses.PipelineResponse{Workers:16, BatchSize:125, BatchDelay:50}, + Pipeline: responses.PipelineResponse{Workers:10, BatchSize:125, BatchDelay:50}, Jvm: responses.JvmResponse{ - Threads: struct { Count int "json:\"count\""; PeakCount int "json:\"peak_count\"" }{Count:60, PeakCount:60}, + Threads: struct { Count int "json:\"count\""; PeakCount int "json:\"peak_count\"" }{Count:65, PeakCount:65}, Mem: struct { HeapUsedPercent int "json:\"heap_used_percent\""; HeapCommittedInBytes int64 "json:\"heap_committed_in_bytes\""; HeapMaxInBytes int64 "json:\"heap_max_in_bytes\""; HeapUsedInBytes int64 "json:\"heap_used_in_bytes\""; NonHeapUsedInBytes int64 "json:\"non_heap_used_in_bytes\""; NonHeapCommittedInBytes int64 "json:\"non_heap_committed_in_bytes\""; Pools struct { Young responses.PoolResponse "json:\"young\""; Old responses.PoolResponse "json:\"old\""; Survivor responses.PoolResponse "json:\"survivor\"" } "json:\"pools\"" }{ - HeapUsedPercent: 27, + HeapUsedPercent: 36, HeapCommittedInBytes: 1073741824, HeapMaxInBytes: 1073741822, - HeapUsedInBytes: 294044784, - NonHeapUsedInBytes: 147703688, - NonHeapCommittedInBytes: 155189248, + HeapUsedInBytes: 395837440, + NonHeapUsedInBytes: 172607568, + NonHeapCommittedInBytes: 183304192, Pools: struct { Young responses.PoolResponse "json:\"young\""; Old responses.PoolResponse "json:\"old\""; Survivor responses.PoolResponse "json:\"survivor\"" }{ - Young: responses.PoolResponse{PeakMaxInBytes:-1, MaxInBytes:-1, CommittedInBytes:346030080, PeakUsedInBytes:326107136, UsedInBytes:180355072}, - Old: responses.PoolResponse{PeakMaxInBytes:1073741824, MaxInBytes:1073741824, CommittedInBytes:687865856, PeakUsedInBytes:73986560, UsedInBytes:73986560}, - Survivor: responses.PoolResponse{PeakMaxInBytes:-1, MaxInBytes:-1, CommittedInBytes:39845888, PeakUsedInBytes:39703152, UsedInBytes:39703152}, + Young: responses.PoolResponse{PeakMaxInBytes:-1, MaxInBytes:-1, CommittedInBytes:668991488, PeakUsedInBytes:351272960, UsedInBytes:257949696}, + Old: responses.PoolResponse{PeakMaxInBytes:1073741824, MaxInBytes:1073741824, CommittedInBytes:397410304, PeakUsedInBytes:132644864, UsedInBytes:130547712}, + Survivor: responses.PoolResponse{PeakMaxInBytes:-1, MaxInBytes:-1, CommittedInBytes:7340032, PeakUsedInBytes:47185920, UsedInBytes:7340032}, }, }, Gc: struct { Collectors struct { Young responses.CollectorResponse "json:\"young\""; Old responses.CollectorResponse "json:\"old\"" } "json:\"collectors\"" }{ Collectors: struct { Young responses.CollectorResponse "json:\"young\""; Old responses.CollectorResponse "json:\"old\"" }{ - Young: responses.CollectorResponse{CollectionCount:8, CollectionTimeInMillis:224}, + Young: responses.CollectorResponse{CollectionCount:11, CollectionTimeInMillis:110}, Old: responses.CollectorResponse{}, }, }, - UptimeInMillis: 53120, + UptimeInMillis: 56226, }, Process: responses.ProcessResponse{ - OpenFileDescriptors: 98, - PeakOpenFileDescriptors: 98, + OpenFileDescriptors: 108, + PeakOpenFileDescriptors: 109, MaxFileDescriptors: 1048576, - Mem: struct { TotalVirtualInBytes int64 "json:\"total_virtual_in_bytes\"" }{TotalVirtualInBytes:9305346048}, + Mem: struct { TotalVirtualInBytes int64 "json:\"total_virtual_in_bytes\"" }{TotalVirtualInBytes:8282685440}, CPU: struct { TotalInMillis int64 "json:\"total_in_millis\""; Percent int "json:\"percent\""; LoadAverage struct { OneM float64 "json:\"1m\""; FiveM float64 "json:\"5m\""; FifteenM float64 "json:\"15m\"" } "json:\"load_average\"" }{ - TotalInMillis: 135300, + TotalInMillis: 35730, Percent: 0, - LoadAverage: struct { OneM float64 "json:\"1m\""; FiveM float64 "json:\"5m\""; FifteenM float64 "json:\"15m\"" }{OneM:3.79, FiveM:1.29, FifteenM:0.46}, + LoadAverage: struct { OneM float64 "json:\"1m\""; FiveM float64 "json:\"5m\""; FifteenM float64 "json:\"15m\"" }{OneM:2.25, FiveM:0.68, FifteenM:0.24}, }, }, - Events: responses.EventsResponse{In:4001, Filtered:10, Out:2, DurationInMillis:5, QueuePushDurationInMillis:7}, + Events: responses.EventsResponse{In:3751, Filtered:1250, Out:1250, DurationInMillis:494960, QueuePushDurationInMillis:49451}, Flow: responses.FlowResponse{ - InputThroughput: struct { Current float64 "json:\"current\""; Lifetime float64 "json:\"lifetime\"" }{Current:1, Lifetime:117.4}, - FilterThroughput: struct { Current float64 "json:\"current\""; Lifetime float64 "json:\"lifetime\"" }{Current:2.1, Lifetime:3.2}, - OutputThroughput: struct { Current float64 "json:\"current\""; Lifetime float64 "json:\"lifetime\"" }{Current:4.3, Lifetime:5.4}, - QueueBackpressure: struct { Current float64 "json:\"current\""; Lifetime float64 "json:\"lifetime\"" }{Current:6.5, Lifetime:7.6}, - WorkerConcurrency: struct { Current float64 "json:\"current\""; Lifetime float64 "json:\"lifetime\"" }{Current:8.7, Lifetime:9.8}, - }, - Reloads: responses.ReloadResponse{}, - Os: responses.OsResponse{ - Cgroup: struct { Cpu struct { CfsPeriodMicros int64 "json:\"cfs_period_micros\""; CfsQuotaMicros int64 "json:\"cfs_quota_micros\""; Stat struct { TimeThrottledNanos int64 "json:\"time_throttled_nanos\""; NumberOfTimesThrottled int64 "json:\"number_of_times_throttled\""; NumberOfElapsedPeriods int64 "json:\"number_of_elapsed_periods\"" } "json:\"stat\""; ControlGroup string "json:\"control_group\"" } "json:\"cpu\""; Cpuacct struct { UsageNanos int64 "json:\"usage_nanos\""; ControlGroup string "json:\"control_group\"" } "json:\"cpuacct\"" }{ - Cpu: struct { CfsPeriodMicros int64 "json:\"cfs_period_micros\""; CfsQuotaMicros int64 "json:\"cfs_quota_micros\""; Stat struct { TimeThrottledNanos int64 "json:\"time_throttled_nanos\""; NumberOfTimesThrottled int64 "json:\"number_of_times_throttled\""; NumberOfElapsedPeriods int64 "json:\"number_of_elapsed_periods\"" } "json:\"stat\""; ControlGroup string "json:\"control_group\"" }{ - CfsPeriodMicros: 100000, - CfsQuotaMicros: -1, - Stat: struct { TimeThrottledNanos int64 "json:\"time_throttled_nanos\""; NumberOfTimesThrottled int64 "json:\"number_of_times_throttled\""; NumberOfElapsedPeriods int64 "json:\"number_of_elapsed_periods\"" }{}, - ControlGroup: "/", - }, - Cpuacct: struct { UsageNanos int64 "json:\"usage_nanos\""; ControlGroup string "json:\"control_group\"" }{UsageNanos:161531487900, ControlGroup:"/"}, - }, + InputThroughput: struct { Current float64 "json:\"current\""; Lifetime float64 "json:\"lifetime\"" }{Current:0, Lifetime:73.9}, + FilterThroughput: struct { Current float64 "json:\"current\""; Lifetime float64 "json:\"lifetime\"" }{Current:0, Lifetime:24.63}, + OutputThroughput: struct { Current float64 "json:\"current\""; Lifetime float64 "json:\"lifetime\"" }{Current:0, Lifetime:24.63}, + QueueBackpressure: struct { Current float64 "json:\"current\""; Lifetime float64 "json:\"lifetime\"" }{Current:1, Lifetime:0.9743}, + WorkerConcurrency: struct { Current float64 "json:\"current\""; Lifetime float64 "json:\"lifetime\"" }{Current:10, Lifetime:9.752}, }, + Reloads: responses.ReloadResponse{}, + Os: responses.OsResponse{}, Queue: responses.QueueResponse{}, Pipelines: { ".monitoring-logstash": { @@ -73,15 +63,36 @@ responses.NodeStatsResponse{ Flow: responses.FlowResponse{}, Plugins: struct { Inputs []struct { ID string "json:\"id\""; Name string "json:\"name\""; Events struct { Out int "json:\"out\""; QueuePushDurationInMillis int "json:\"queue_push_duration_in_millis\"" } "json:\"events\"" } "json:\"inputs\""; Codecs []struct { ID string "json:\"id\""; Name string "json:\"name\""; Decode struct { Out int "json:\"out\""; WritesIn int "json:\"writes_in\""; DurationInMillis int "json:\"duration_in_millis\"" } "json:\"decode\""; Encode struct { WritesIn int "json:\"writes_in\""; DurationInMillis int "json:\"duration_in_millis\"" } "json:\"encode\"" } "json:\"codecs\""; Filters []struct { ID string "json:\"id\""; Name string "json:\"name\""; Events struct { Out int "json:\"out\""; In int "json:\"in\""; DurationInMillis int "json:\"duration_in_millis\"" } "json:\"events\"" } "json:\"filters\""; Outputs []struct { ID string "json:\"id\""; Name string "json:\"name\""; Events struct { Out int "json:\"out\""; In int "json:\"in\""; DurationInMillis int "json:\"duration_in_millis\"" } "json:\"events\""; Documents struct { Successes int "json:\"successes\""; NonRetryableFailures int "json:\"non_retryable_failures\"" } "json:\"documents\""; BulkRequests struct { WithErrors int "json:\"with_errors\""; Responses map[string]int "json:\"responses\"" } "json:\"bulk_requests\"" } "json:\"outputs\"" }{ Inputs: { + { + ID: "9a9bed30135e19c8047fe6aa0588b70b15280fb9161fea8ed8e7368e1fb1e6d3", + Name: "", + Events: struct { Out int "json:\"out\""; QueuePushDurationInMillis int "json:\"queue_push_duration_in_millis\"" }{}, + }, }, Codecs: { }, Filters: { }, Outputs: { + { + ID: "e7aab1e17ec42d6573f3be4b8fce17c5dc69db8473f9505f386b1160434b141b", + Name: "", + Events: struct { Out int "json:\"out\""; In int "json:\"in\""; DurationInMillis int "json:\"duration_in_millis\"" }{}, + Documents: struct { Successes int "json:\"successes\""; NonRetryableFailures int "json:\"non_retryable_failures\"" }{}, + BulkRequests: struct { WithErrors int "json:\"with_errors\""; Responses map[string]int "json:\"responses\"" }{}, + }, + }, + }, + Reloads: responses.PipelineReloadResponse{ + LastFailureTimestamp: time.Date(2023, time.April, 20, 20, 0, 32, 437218256, time.UTC), + Successes: 3, + Failures: 1, + LastSuccessTimestamp: time.Date(2023, time.April, 20, 22, 30, 32, 437218256, time.UTC), + LastError: responses.LastError{ + Message: "No configuration found in the configured sources.", + Backtrace: {"org/logstash/execution/AbstractPipelineExt.java:151:in `reload_pipeline'", "/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:181:in `block in reload_pipeline'", "/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/stud-0.0.23/lib/stud/task.rb:24:in `block in initialize'"}, }, }, - Reloads: responses.PipelineReloadResponse{}, Queue: struct { Type string "json:\"type\""; EventsCount int64 "json:\"events_count\""; QueueSizeInBytes int64 "json:\"queue_size_in_bytes\""; MaxQueueSizeInBytes int64 "json:\"max_queue_size_in_bytes\"" }{}, DeadLetterQueue: struct { MaxQueueSizeInBytes int "json:\"max_queue_size_in_bytes\""; QueueSizeInBytes int64 "json:\"queue_size_in_bytes\""; DroppedEvents int64 "json:\"dropped_events\""; ExpiredEvents int64 "json:\"expired_events\""; StoragePolicy string "json:\"storage_policy\"" }{}, Hash: "", @@ -89,80 +100,97 @@ responses.NodeStatsResponse{ }, "main": { Monitoring: responses.PipelineLogstashMonitoringResponse{}, - Events: responses.EventsResponse{In:4001, Filtered:0, Out:0, DurationInMillis:0, QueuePushDurationInMillis:0}, + Events: responses.EventsResponse{In:3751, Filtered:1250, Out:1250, DurationInMillis:495018, QueuePushDurationInMillis:49455}, Flow: responses.FlowResponse{ - InputThroughput: struct { Current float64 "json:\"current\""; Lifetime float64 "json:\"lifetime\"" }{Current:6.7, Lifetime:124}, - FilterThroughput: struct { Current float64 "json:\"current\""; Lifetime float64 "json:\"lifetime\"" }{Current:7.8, Lifetime:8.9}, - OutputThroughput: struct { Current float64 "json:\"current\""; Lifetime float64 "json:\"lifetime\"" }{Current:2.3, Lifetime:3.4}, - QueueBackpressure: struct { Current float64 "json:\"current\""; Lifetime float64 "json:\"lifetime\"" }{Current:1.1, Lifetime:2.2}, - WorkerConcurrency: struct { Current float64 "json:\"current\""; Lifetime float64 "json:\"lifetime\"" }{Current:4.5, Lifetime:5.6}, + InputThroughput: struct { Current float64 "json:\"current\""; Lifetime float64 "json:\"lifetime\"" }{Current:0, Lifetime:74.88}, + FilterThroughput: struct { Current float64 "json:\"current\""; Lifetime float64 "json:\"lifetime\"" }{Current:0, Lifetime:24.95}, + OutputThroughput: struct { Current float64 "json:\"current\""; Lifetime float64 "json:\"lifetime\"" }{Current:0, Lifetime:24.95}, + QueueBackpressure: struct { Current float64 "json:\"current\""; Lifetime float64 "json:\"lifetime\"" }{Current:1, Lifetime:0.9872}, + WorkerConcurrency: struct { Current float64 "json:\"current\""; Lifetime float64 "json:\"lifetime\"" }{Current:10, Lifetime:9.882}, }, Plugins: struct { Inputs []struct { ID string "json:\"id\""; Name string "json:\"name\""; Events struct { Out int "json:\"out\""; QueuePushDurationInMillis int "json:\"queue_push_duration_in_millis\"" } "json:\"events\"" } "json:\"inputs\""; Codecs []struct { ID string "json:\"id\""; Name string "json:\"name\""; Decode struct { Out int "json:\"out\""; WritesIn int "json:\"writes_in\""; DurationInMillis int "json:\"duration_in_millis\"" } "json:\"decode\""; Encode struct { WritesIn int "json:\"writes_in\""; DurationInMillis int "json:\"duration_in_millis\"" } "json:\"encode\"" } "json:\"codecs\""; Filters []struct { ID string "json:\"id\""; Name string "json:\"name\""; Events struct { Out int "json:\"out\""; In int "json:\"in\""; DurationInMillis int "json:\"duration_in_millis\"" } "json:\"events\"" } "json:\"filters\""; Outputs []struct { ID string "json:\"id\""; Name string "json:\"name\""; Events struct { Out int "json:\"out\""; In int "json:\"in\""; DurationInMillis int "json:\"duration_in_millis\"" } "json:\"events\""; Documents struct { Successes int "json:\"successes\""; NonRetryableFailures int "json:\"non_retryable_failures\"" } "json:\"documents\""; BulkRequests struct { WithErrors int "json:\"with_errors\""; Responses map[string]int "json:\"responses\"" } "json:\"bulk_requests\"" } "json:\"outputs\"" }{ Inputs: { { - ID: "c75c0c6f97fd2c8605b95a5b2694fdae97189fe49553787a923faeaa3342c54a", + ID: "5ee0ea3d45c32bab3b41963bd900e758ba6e193a11079649302574c706fd5e2f", + Name: "dead_letter_queue", + Events: struct { Out int "json:\"out\""; QueuePushDurationInMillis int "json:\"queue_push_duration_in_millis\"" }{}, + }, + { + ID: "95bb3e4f2a40f87147b6ab5035e08ba31858eace7604a57d2e719db790097222", Name: "generator", - Events: struct { Out int "json:\"out\""; QueuePushDurationInMillis int "json:\"queue_push_duration_in_millis\"" }{Out:4001, QueuePushDurationInMillis:0}, + Events: struct { Out int "json:\"out\""; QueuePushDurationInMillis int "json:\"queue_push_duration_in_millis\"" }{Out:3751, QueuePushDurationInMillis:49454}, }, }, Codecs: { { - ID: "plain_d7fb272d-75f5-4e38-bf56-c4c798bea2d1", + ID: "plain_4ffdd0bf-b707-419b-b425-d3986a43c35a", Name: "plain", Decode: struct { Out int "json:\"out\""; WritesIn int "json:\"writes_in\""; DurationInMillis int "json:\"duration_in_millis\"" }{}, Encode: struct { WritesIn int "json:\"writes_in\""; DurationInMillis int "json:\"duration_in_millis\"" }{}, }, { - ID: "plain_35cb2d25-0d8d-441a-a714-82121715864d", + ID: "plain_b4b163b6-effd-454a-9605-c3f8ef0cde5e", Name: "plain", - Decode: struct { Out int "json:\"out\""; WritesIn int "json:\"writes_in\""; DurationInMillis int "json:\"duration_in_millis\"" }{Out:4001, WritesIn:4001, DurationInMillis:57}, + Decode: struct { Out int "json:\"out\""; WritesIn int "json:\"writes_in\""; DurationInMillis int "json:\"duration_in_millis\"" }{}, + Encode: struct { WritesIn int "json:\"writes_in\""; DurationInMillis int "json:\"duration_in_millis\"" }{}, + }, + { + ID: "plain-codec-001", + Name: "plain", + Decode: struct { Out int "json:\"out\""; WritesIn int "json:\"writes_in\""; DurationInMillis int "json:\"duration_in_millis\"" }{}, Encode: struct { WritesIn int "json:\"writes_in\""; DurationInMillis int "json:\"duration_in_millis\"" }{}, }, }, Filters: { { - ID: "1721e7c39758977b227be1d9334f0752555f39c873b8b86a3df8546f64059112", - Name: "json", - Events: struct { Out int "json:\"out\""; In int "json:\"in\""; DurationInMillis int "json:\"duration_in_millis\"" }{Out:2000, In:2000, DurationInMillis:716}, + ID: "prune-http-input-fields", + Name: "prune", + Events: struct { Out int "json:\"out\""; In int "json:\"in\""; DurationInMillis int "json:\"duration_in_millis\"" }{Out:1250, In:1250, DurationInMillis:127}, + }, + { + ID: "ca953dac49c8fd3b00ba8275af10f9c6bcd9ca95755cd7892952966c5a13d427", + Name: "ruby", + Events: struct { Out int "json:\"out\""; In int "json:\"in\""; DurationInMillis int "json:\"duration_in_millis\"" }{Out:1250, In:2500, DurationInMillis:489610}, }, { - ID: "drop_b8ed8ea8c0ace91d6b617f6c8a5153141183c35a330de014182825dbceeade00", + ID: "drop-non-existent", Name: "drop", - Events: struct { Out int "json:\"out\""; In int "json:\"in\""; DurationInMillis int "json:\"duration_in_millis\"" }{Out:300, In:330, DurationInMillis:333}, + Events: struct { Out int "json:\"out\""; In int "json:\"in\""; DurationInMillis int "json:\"duration_in_millis\"" }{}, + }, + { + ID: "json-filter", + Name: "json", + Events: struct { Out int "json:\"out\""; In int "json:\"in\""; DurationInMillis int "json:\"duration_in_millis\"" }{Out:1250, In:1250, DurationInMillis:214}, + }, + { + ID: "mutate-path-001", + Name: "mutate", + Events: struct { Out int "json:\"out\""; In int "json:\"in\""; DurationInMillis int "json:\"duration_in_millis\"" }{Out:1250, In:1250, DurationInMillis:170}, }, { - ID: "drop_e2e0f559b7292f788693f9f318185d5c1d30127870ca8f0e608b11d9dc560079", + ID: "drop-80-percent", Name: "drop", - Events: struct { Out int "json:\"out\""; In int "json:\"in\""; DurationInMillis int "json:\"duration_in_millis\"" }{Out:800, In:880, DurationInMillis:888}, + Events: struct { Out int "json:\"out\""; In int "json:\"in\""; DurationInMillis int "json:\"duration_in_millis\"" }{}, }, }, Outputs: { { - ID: "45554a51a53a57f5dbba7d26b65aad526147453a895529f3d4698c8fd88692ef", + ID: "81d12b78dc90935689721a2220cbd7ad2f75910cdafdc01b5b77f6ca8f356cef", Name: "elasticsearch", - Events: struct { Out int "json:\"out\""; In int "json:\"in\""; DurationInMillis int "json:\"duration_in_millis\"" }{Out:0, In:2000, DurationInMillis:0}, - Documents: struct { Successes int "json:\"successes\""; NonRetryableFailures int "json:\"non_retryable_failures\"" }{Successes:1337, NonRetryableFailures:87}, + Events: struct { Out int "json:\"out\""; In int "json:\"in\""; DurationInMillis int "json:\"duration_in_millis\"" }{Out:1250, In:1250, DurationInMillis:4887}, + Documents: struct { Successes int "json:\"successes\""; NonRetryableFailures int "json:\"non_retryable_failures\"" }{Successes:1250, NonRetryableFailures:0}, BulkRequests: struct { WithErrors int "json:\"with_errors\""; Responses map[string]int "json:\"responses\"" }{ - WithErrors: 87, - Responses: {"200":87}, + WithErrors: 0, + Responses: {"200":10}, }, }, }, }, - Reloads: responses.PipelineReloadResponse{ - LastFailureTimestamp: time.Date(2023, time.April, 20, 20, 0, 32, 437218256, time.UTC), - Successes: 3, - Failures: 1, - LastSuccessTimestamp: time.Date(2023, time.April, 20, 22, 30, 32, 437218256, time.UTC), - LastError: responses.LastError{ - Message: "No configuration found in the configured sources.", - Backtrace: {"org/logstash/execution/AbstractPipelineExt.java:151:in `reload_pipeline'", "/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:181:in `block in reload_pipeline'", "/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/stud-0.0.23/lib/stud/task.rb:24:in `block in initialize'"}, - }, - }, + Reloads: responses.PipelineReloadResponse{}, Queue: struct { Type string "json:\"type\""; EventsCount int64 "json:\"events_count\""; QueueSizeInBytes int64 "json:\"queue_size_in_bytes\""; MaxQueueSizeInBytes int64 "json:\"max_queue_size_in_bytes\"" }{Type:"memory", EventsCount:0, QueueSizeInBytes:0, MaxQueueSizeInBytes:0}, - DeadLetterQueue: struct { MaxQueueSizeInBytes int "json:\"max_queue_size_in_bytes\""; QueueSizeInBytes int64 "json:\"queue_size_in_bytes\""; DroppedEvents int64 "json:\"dropped_events\""; ExpiredEvents int64 "json:\"expired_events\""; StoragePolicy string "json:\"storage_policy\"" }{MaxQueueSizeInBytes:47244640256, QueueSizeInBytes:1, DroppedEvents:0, ExpiredEvents:0, StoragePolicy:"drop_newer"}, - Hash: "a73729cc9c29203931db21553c5edba063820a7e40d16cb5053be75cc3811a17", - EphemeralID: "a5c63d09-1ba6-4d67-90a5-075f468a7ab0", + DeadLetterQueue: struct { MaxQueueSizeInBytes int "json:\"max_queue_size_in_bytes\""; QueueSizeInBytes int64 "json:\"queue_size_in_bytes\""; DroppedEvents int64 "json:\"dropped_events\""; ExpiredEvents int64 "json:\"expired_events\""; StoragePolicy string "json:\"storage_policy\"" }{MaxQueueSizeInBytes:1073741824, QueueSizeInBytes:1, DroppedEvents:0, ExpiredEvents:0, StoragePolicy:"drop_newer"}, + Hash: "d30c4ff4da9fdb1a6b06ee390df1336aa80cc5ce6582d316af3dc0695af2d82e", + EphemeralID: "31caf4d6-162d-4eeb-bc04-411ae2e996f1", }, }, }