From 88d0a0eb9dc437b289ae9e3e9ea74a4f71db57d5 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Wed, 10 Jan 2024 10:52:16 -0800 Subject: [PATCH 01/32] Add a add_session_metadata auditbeat processor This processor will enrich process events with additional infomation needed to enable session view in Kibana. This processor can be run on Linux systems, and will use eBPF to enrich auditd events for process exec and exit events. The additional fields that will be added are information on process parent, session leader and process group leader. --- NOTICE.txt | 207 ++++++-- auditbeat/cmd/root.go | 3 + dev-tools/notice/overrides.json | 1 + go.mod | 5 +- go.sum | 15 +- .../add_session_metadata.go | 196 +++++++ .../processors/add_session_metadata/config.go | 20 + .../add_session_metadata/pkg/processdb/db.go | 51 ++ .../pkg/processdb/db_test.go | 18 + .../pkg/processdb/simple.go | 482 ++++++++++++++++++ .../pkg/processdb/simple_test.go | 55 ++ .../add_session_metadata/pkg/procfs/mock.go | 40 ++ .../add_session_metadata/pkg/procfs/procfs.go | 250 +++++++++ .../pkg/timeutils/time.go | 62 +++ .../pkg/timeutils/time_test.go | 22 + .../provider/ebpf_provider/ebpf_provider.go | 159 ++++++ .../add_session_metadata/provider/provider.go | 13 + .../add_session_metadata/types/events.go | 94 ++++ .../add_session_metadata/types/process.go | 367 +++++++++++++ 19 files changed, 2013 insertions(+), 47 deletions(-) create mode 100644 x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go create mode 100644 x-pack/auditbeat/processors/add_session_metadata/config.go create mode 100644 x-pack/auditbeat/processors/add_session_metadata/pkg/processdb/db.go create mode 100644 x-pack/auditbeat/processors/add_session_metadata/pkg/processdb/db_test.go create mode 100644 x-pack/auditbeat/processors/add_session_metadata/pkg/processdb/simple.go create mode 100644 x-pack/auditbeat/processors/add_session_metadata/pkg/processdb/simple_test.go create mode 100644 x-pack/auditbeat/processors/add_session_metadata/pkg/procfs/mock.go create mode 100644 x-pack/auditbeat/processors/add_session_metadata/pkg/procfs/procfs.go create mode 100644 x-pack/auditbeat/processors/add_session_metadata/pkg/timeutils/time.go create mode 100644 x-pack/auditbeat/processors/add_session_metadata/pkg/timeutils/time_test.go create mode 100644 x-pack/auditbeat/processors/add_session_metadata/provider/ebpf_provider/ebpf_provider.go create mode 100644 x-pack/auditbeat/processors/add_session_metadata/provider/provider.go create mode 100644 x-pack/auditbeat/processors/add_session_metadata/types/events.go create mode 100644 x-pack/auditbeat/processors/add_session_metadata/types/process.go diff --git a/NOTICE.txt b/NOTICE.txt index 1f4940889c6a..be9e8807439a 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -12266,6 +12266,32 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +-------------------------------------------------------------------------------- +Dependency : github.com/elastic/ebpfevents +Version: v0.1.0 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/elastic/ebpfevents@v0.1.0/LICENSE.txt: + +The https://github.com/elastic/ebpfevents repository contains source code under +various licenses: + +- Source code in the 'headers/bpf' directory, is dual-licensed under the GNU Lesser General + Public License version 2.1 (LICENSES/LGPL-2.1-only.txt) OR BSD-2-Clause license + (LICENSES/BSD-2-Clause.txt) + +- Source code in the 'ebpf' submodule is licensed with multiple licenses. Read more at + https://github.com/elastic/ebpf/blob/main/LICENSE.txt. + +- The binary files 'bpf_bpfel_x86.o' and 'bpf_bpfel_amd64.o' are compiled + from dual-licensed GPL-2.0-only OR BSD-2-Clause licensed code, and are distributed with + the GPL-2.0-only License (LICENSES/GPL-2.0-only.txt). + +- Source code not listed in the previous points is licensed under the Apache License, + version 2 (LICENSES/Apache-2.0.txt). + + -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-autodiscover Version: v0.6.6 @@ -21129,6 +21155,37 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +-------------------------------------------------------------------------------- +Dependency : github.com/mohae/deepcopy +Version: v0.0.0-20170929034955-c48cc78d4826 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/mohae/deepcopy@v0.0.0-20170929034955-c48cc78d4826/LICENSE: + +The MIT License (MIT) + +Copyright (c) 2014 Joel + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + -------------------------------------------------------------------------------- Dependency : github.com/olekukonko/tablewriter Version: v0.0.5 @@ -22600,6 +22657,45 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +-------------------------------------------------------------------------------- +Dependency : github.com/tklauser/go-sysconf +Version: v0.3.10 +Licence type (autodetected): BSD-3-Clause +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/tklauser/go-sysconf@v0.3.10/LICENSE: + +BSD 3-Clause License + +Copyright (c) 2018-2021, Tobias Klauser +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + -------------------------------------------------------------------------------- Dependency : github.com/tsg/go-daemon Version: v0.0.0-20200207173439-e704b93fd89b @@ -34364,6 +34460,39 @@ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +-------------------------------------------------------------------------------- +Dependency : github.com/cilium/ebpf +Version: v0.12.3 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/cilium/ebpf@v0.12.3/LICENSE: + +MIT License + +Copyright (c) 2017 Nathan Sweet +Copyright (c) 2018, 2019 Cloudflare +Copyright (c) 2019 Authors of Cilium + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + -------------------------------------------------------------------------------- Dependency : github.com/codegangsta/inject Version: v0.0.0-20150114235600-33e0aa1cb7c0 @@ -36369,11 +36498,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : github.com/frankban/quicktest -Version: v1.14.3 +Version: v1.14.5 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/frankban/quicktest@v1.14.3/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/frankban/quicktest@v1.14.5/LICENSE: MIT License @@ -36398,6 +36527,37 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +-------------------------------------------------------------------------------- +Dependency : github.com/go-faker/faker/v4 +Version: v4.2.0 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/go-faker/faker/v4@v4.2.0/LICENSE: + +MIT License + +Copyright (c) 2017 Iman Tumorang + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + -------------------------------------------------------------------------------- Dependency : github.com/go-logfmt/logfmt Version: v0.5.1 @@ -43840,11 +44000,11 @@ OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/kr/pretty -Version: v0.3.0 +Version: v0.3.1 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/kr/pretty@v0.3.0/License: +Contents of probable licence file $GOMODCACHE/github.com/kr/pretty@v0.3.1/License: Copyright 2012 Keith Rarick @@ -48514,45 +48674,6 @@ IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. --------------------------------------------------------------------------------- -Dependency : github.com/tklauser/go-sysconf -Version: v0.3.10 -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/tklauser/go-sysconf@v0.3.10/LICENSE: - -BSD 3-Clause License - -Copyright (c) 2018-2021, Tobias Klauser -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -* Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -------------------------------------------------------------------------------- Dependency : github.com/tklauser/numcpus Version: v0.4.0 diff --git a/auditbeat/cmd/root.go b/auditbeat/cmd/root.go index 0ddc7b8674da..4c8c723c4539 100644 --- a/auditbeat/cmd/root.go +++ b/auditbeat/cmd/root.go @@ -30,6 +30,9 @@ import ( "github.com/elastic/beats/v7/metricbeat/beater" "github.com/elastic/beats/v7/metricbeat/mb/module" "github.com/elastic/elastic-agent-libs/mapstr" + + // Import processors + _ "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata" ) const ( diff --git a/dev-tools/notice/overrides.json b/dev-tools/notice/overrides.json index 1484fcde52a0..eee18acc0de5 100644 --- a/dev-tools/notice/overrides.json +++ b/dev-tools/notice/overrides.json @@ -17,3 +17,4 @@ {"name": "github.com/awslabs/kinesis-aggregation/go/v2", "licenceType": "Apache-2.0", "url": "https://github.com/awslabs/kinesis-aggregation/blob/master/LICENSE.txt"} {"name": "github.com/dnaeon/go-vcr", "licenceType": "BSD-2-Clause"} {"name": "github.com/JohnCGriffin/overflow", "licenceType": "MIT"} +{"name": "github.com/elastic/ebpfevents", "licenceType": "Apache-2.0"} diff --git a/go.mod b/go.mod index 786e0cdd8b57..a35dea2f998c 100644 --- a/go.mod +++ b/go.mod @@ -200,6 +200,7 @@ require ( github.com/aws/smithy-go v1.13.5 github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5 github.com/elastic/bayeux v1.0.5 + github.com/elastic/ebpfevents v0.1.0 github.com/elastic/elastic-agent-autodiscover v0.6.6 github.com/elastic/elastic-agent-libs v0.7.3 github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 @@ -214,11 +215,13 @@ require ( github.com/gorilla/mux v1.8.0 github.com/icholy/digest v0.1.22 github.com/lestrrat-go/jwx/v2 v2.0.11 + github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 github.com/otiai10/copy v1.12.0 github.com/pierrec/lz4/v4 v4.1.16 github.com/pkg/xattr v0.4.9 github.com/sergi/go-diff v1.3.1 github.com/shirou/gopsutil/v3 v3.22.10 + github.com/tklauser/go-sysconf v0.3.10 go.elastic.co/apm/module/apmelasticsearch/v2 v2.4.7 go.elastic.co/apm/module/apmhttp/v2 v2.4.7 go.elastic.co/apm/v2 v2.4.7 @@ -264,6 +267,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash v1.1.0 // indirect + github.com/cilium/ebpf v0.12.3 // indirect github.com/cyphar/filepath-securejoin v0.2.4 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect @@ -352,7 +356,6 @@ require ( github.com/sirupsen/logrus v1.9.0 // indirect github.com/stoewer/go-strcase v1.2.0 // indirect github.com/stretchr/objx v0.5.0 // indirect - github.com/tklauser/go-sysconf v0.3.10 // indirect github.com/tklauser/numcpus v0.4.0 // indirect github.com/urso/diag v0.0.0-20200210123136-21b3cc8eb797 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect diff --git a/go.sum b/go.sum index 1fa1bc366e26..20a14893850a 100644 --- a/go.sum +++ b/go.sum @@ -427,6 +427,8 @@ github.com/cilium/ebpf v0.0.0-20200702112145-1c8d4c9ef775/go.mod h1:7cR51M8ViRLI github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= +github.com/cilium/ebpf v0.12.3 h1:8ht6F9MquybnY97at+VDZb3eQQr8ev79RueWeVaEcG4= +github.com/cilium/ebpf v0.12.3/go.mod h1:TctK1ivibvI3znr66ljgi4hqOT8EYQjz1KWBfb1UVgM= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= @@ -655,6 +657,8 @@ github.com/elastic/bayeux v1.0.5 h1:UceFq01ipmT3S8DzFK+uVAkbCdiPR0Bqei8qIGmUeY0= github.com/elastic/bayeux v1.0.5/go.mod h1:CSI4iP7qeo5MMlkznGvYKftp8M7qqP/3nzmVZoXHY68= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY= +github.com/elastic/ebpfevents v0.1.0 h1:Kr62fVcDSrPYpwsW3FXUOcmImHqbiRfwmb6fZOw5PgI= +github.com/elastic/ebpfevents v0.1.0/go.mod h1:o21z5xup/9dK8u0Hg9bZRflSqqj1Zu5h2dg2hSTcUPQ= github.com/elastic/elastic-agent-autodiscover v0.6.6 h1:P1y0dDpbhJc7Uw/xe85irPEad4Vljygc+y4iSxtqW7A= github.com/elastic/elastic-agent-autodiscover v0.6.6/go.mod h1:chulyCAyZb/njMHgzkhC/yWnt8v/Y6eCRUhmFVnsA5o= github.com/elastic/elastic-agent-client/v7 v7.6.0 h1:FEn6FjzynW4TIQo5G096Tr7xYK/P5LY9cSS6wRbXZTc= @@ -752,8 +756,8 @@ github.com/foxcpp/go-mockdns v0.0.0-20201212160233-ede2f9158d15/go.mod h1:tPg4cp github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= -github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= -github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= +github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA= +github.com/frankban/quicktest v1.14.5/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA= github.com/gabriel-vasile/mimetype v1.4.1/go.mod h1:05Vi0w3Y9c/lNvJOdmIwvrrAhX3rYhfQQCaf9VJcv7M= github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= @@ -765,6 +769,8 @@ github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0 github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= github.com/go-chi/chi v4.1.0+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= +github.com/go-faker/faker/v4 v4.2.0 h1:dGebOupKwssrODV51E0zbMrv5e2gO9VWSLNC1WDCpWg= +github.com/go-faker/faker/v4 v4.2.0/go.mod h1:F/bBy8GH9NxOxMInug5Gx4WYeG6fHJZ8Ol/dhcpRub4= github.com/go-fonts/dejavu v0.1.0/go.mod h1:4Wt4I4OU2Nq9asgDCteaAaWZOV24E+0/Pwo0gppep4g= github.com/go-fonts/latin-modern v0.2.0/go.mod h1:rQVLdDMK+mK1xscDwsqM5J8U2jrRa3T0ecnM9pNujks= github.com/go-fonts/liberation v0.1.1/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= @@ -1336,8 +1342,9 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -1481,6 +1488,8 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/montanaflynn/stats v0.6.6/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= diff --git a/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go b/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go new file mode 100644 index 000000000000..a412662cc6f5 --- /dev/null +++ b/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go @@ -0,0 +1,196 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package add_session_metadata + +import ( + "context" + "fmt" + "reflect" + "strconv" + "time" + + "github.com/elastic/elastic-agent-libs/mapstr" + + "github.com/google/uuid" + + "github.com/elastic/elastic-agent-libs/monitoring" + + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/pkg/processdb" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/pkg/procfs" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/provider" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/provider/ebpf_provider" + + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/processors" + "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/logp" +) + +const ( + processorName = "add_session_metadata" + logName = "processor." + processorName +) + +var reg *monitoring.Registry + +func init() { + processors.RegisterPlugin(processorName, New) + reg = monitoring.Default.NewRegistry(logName, monitoring.DoNotReport) +} + +type addSessionMetadata struct { + config Config + logger *logp.Logger + db processdb.DB + provider provider.Provider +} + +func New(cfg *config.C) (beat.Processor, error) { + c := defaultConfig() + if err := cfg.Unpack(&c); err != nil { + return nil, fmt.Errorf("fail to unpack the %v configuration: %w", processorName, err) + } + + logger := logp.NewLogger(logName) + + ctx := context.TODO() + reader := procfs.NewProcfsReader(*logger) + db := processdb.NewSimpleDB(reader, *logger) + + backfilledPIDs := db.ScrapeProcfs() + logger.Debugf("backfilled %d processes", len(backfilledPIDs)) + + switch c.Backend { + case "ebpf": + p, err := ebpf_provider.NewProvider(ctx, *logger, db) + if err != nil { + return nil, fmt.Errorf("failed to create ebpf provider: %w", err) + } + return &addSessionMetadata{ + config: c, + logger: logger, + db: db, + provider: p, + }, nil + default: + return nil, fmt.Errorf("unknown backend configuration") + } +} + +func (p *addSessionMetadata) Run(ev *beat.Event) (*beat.Event, error) { + _, err := ev.GetValue(p.config.PidField) + if err != nil { + // Do not attempt to enrich events without PID; it's not a supported event + return ev, nil + } + + err = p.provider.UpdateDB(ev) + if err != nil { + return ev, err + } + + result, err := p.enrich(ev) + if err != nil { + return ev, fmt.Errorf("enriching event: %w", err) + } + return result, nil +} + +func (p *addSessionMetadata) String() string { + return fmt.Sprintf("%v=[backend=%s, pid_field=%s, override_fields=%t]", + processorName, p.config.Backend, p.config.PidField, p.config.ReplaceFields) +} + +func (p *addSessionMetadata) enrich(ev *beat.Event) (*beat.Event, error) { + pidIf, err := ev.GetValue(p.config.PidField) + if err != nil { + return nil, err + } + + pid, err := pidToUInt32(pidIf) + if err != nil { + return nil, fmt.Errorf("cannot parse pid field '%s': %w", p.config.PidField, err) + } + + fullProcess, err := p.db.GetProcess(pid) + if err != nil { + return nil, fmt.Errorf("pid %v not found in db: %w", pid, err) + } + + result := ev.Clone() + + processMap := fullProcess.ToMap() + + mapstr.MergeFieldsDeep(result.Fields["process"].(mapstr.M), processMap, true) + + if p.config.ReplaceFields { + if err := p.replaceFields(result); err != nil { + return nil, fmt.Errorf("replace fields: %w", err) + } + } + return result, nil +} + +//pidToUInt32 converts PID value to uint32 +func pidToUInt32(value interface{}) (pid uint32, err error) { + switch v := value.(type) { + case string: + nr, err := strconv.Atoi(v) + if err != nil { + return 0, fmt.Errorf("error converting string to integer: %w", err) + } + pid = uint32(nr) + case uint32: + pid = v + case int, int8, int16, int32, int64: + pid64 := reflect.ValueOf(v).Int() + if pid = uint32(pid64); int64(pid) != pid64 { + return 0, fmt.Errorf("integer out of range: %d", pid64) + } + case uint, uintptr, uint8, uint16, uint64: + pidu64 := reflect.ValueOf(v).Uint() + if pid = uint32(pidu64); uint64(pid) != pidu64 { + return 0, fmt.Errorf("integer out of range: %d", pidu64) + } + default: + return 0, fmt.Errorf("not an integer or string, but %T", v) + } + return pid, nil +} + +//replaceFields replaces event fields with values suitable for session view +func (p *addSessionMetadata) replaceFields(ev *beat.Event) error { + // This will only work with auditd data, but this is entire func is temporary + // It should be removed when session view can work with the original fields. + kind, err := ev.Fields.GetValue("event.kind") + if err != nil { + return err + } + isAuditdEvent, err := ev.Fields.HasKey("auditd") + if err != nil { + return err + } + if kind == "event" && isAuditdEvent { + // process start + syscall, err := ev.Fields.GetValue("auditd.data.syscall") + if err != nil { + return err + } + if syscall == "execve" { + ev.Fields.Put("event.action", []string{"exec", "fork"}) + ev.Fields.Put("event.type", []string{"start"}) + } + + // process end + if syscall == "exit_group" { + ev.Fields.Put("event.action", []string{"end"}) + ev.Fields.Put("event.type", []string{"end"}) + ev.Fields.Put("process.end", time.Now()) + } + + ev.Fields.Put("event.id", uuid.NewString()) + } + return nil +} diff --git a/x-pack/auditbeat/processors/add_session_metadata/config.go b/x-pack/auditbeat/processors/add_session_metadata/config.go new file mode 100644 index 000000000000..71673ec8deb6 --- /dev/null +++ b/x-pack/auditbeat/processors/add_session_metadata/config.go @@ -0,0 +1,20 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package add_session_metadata + +// Config for add_host_metadata processor. +type Config struct { + Backend string `config:"backend"` + ReplaceFields bool `config:"replace_fields"` + PidField string `config:"pid_field"` +} + +func defaultConfig() Config { + return Config{ + Backend: "ebpf", + ReplaceFields: true, + PidField: "process.pid", + } +} diff --git a/x-pack/auditbeat/processors/add_session_metadata/pkg/processdb/db.go b/x-pack/auditbeat/processors/add_session_metadata/pkg/processdb/db.go new file mode 100644 index 000000000000..15857d262b62 --- /dev/null +++ b/x-pack/auditbeat/processors/add_session_metadata/pkg/processdb/db.go @@ -0,0 +1,51 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package processdb + +import ( + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/types" +) + +type DB interface { + InsertFork(fork types.ProcessForkEvent) error + InsertExec(exec types.ProcessExecEvent) error + InsertSetsid(setsid types.ProcessSetsidEvent) error + InsertExit(exit types.ProcessExitEvent) error + GetProcess(pid uint32) (types.Process, error) + ScrapeProcfs() []uint32 +} + +type TtyType int + +const ( + TtyUnknown TtyType = iota + Pts + Tty + TtyConsole +) + +const ( + ptsMinMajor = 136 + ptsMaxMajor = 143 + ttyMajor = 4 + consoleMaxMinor = 63 + ttyMaxMinor = 255 +) + +func getTtyType(major uint16, minor uint16) TtyType { + if major >= ptsMinMajor && major <= ptsMaxMajor { + return Pts + } + + if ttyMajor == major { + if minor <= consoleMaxMinor { + return TtyConsole + } else if minor > consoleMaxMinor && minor <= ttyMaxMinor { + return Tty + } + } + + return TtyUnknown +} diff --git a/x-pack/auditbeat/processors/add_session_metadata/pkg/processdb/db_test.go b/x-pack/auditbeat/processors/add_session_metadata/pkg/processdb/db_test.go new file mode 100644 index 000000000000..1d2428430e2a --- /dev/null +++ b/x-pack/auditbeat/processors/add_session_metadata/pkg/processdb/db_test.go @@ -0,0 +1,18 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package processdb + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetTtyType(t *testing.T) { + assert.Equal(t, TtyConsole, getTtyType(4, 0)) + assert.Equal(t, Pts, getTtyType(136, 0)) + assert.Equal(t, Tty, getTtyType(4, 64)) + assert.Equal(t, TtyUnknown, getTtyType(1000, 1000)) +} diff --git a/x-pack/auditbeat/processors/add_session_metadata/pkg/processdb/simple.go b/x-pack/auditbeat/processors/add_session_metadata/pkg/processdb/simple.go new file mode 100644 index 000000000000..43c56d014449 --- /dev/null +++ b/x-pack/auditbeat/processors/add_session_metadata/pkg/processdb/simple.go @@ -0,0 +1,482 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package processdb + +import ( + "encoding/base64" + "errors" + "fmt" + "math/bits" + "os" + "path" + "sort" + "strconv" + "strings" + "sync" + "time" + + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/pkg/procfs" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/pkg/timeutils" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/types" + "github.com/elastic/elastic-agent-libs/logp" +) + +type Process struct { + Pids types.PidInfo + Creds types.CredInfo + CTty types.TtyDev + Argv []string + Cwd string + Env map[string]string + Filename string +} + +var ( + // The contents of these two files are needed to calculate entity IDs. + // Fail fast on startup if we can't read them. + bootID = mustReadBootID() + pidNsInode = mustReadPidNsInode() + capNames = []string{ + "CAP_CHOWN", // 0 + "CAP_DAC_OVERRIDE", // 1 + "CAP_DAC_READ_SEARCH", // 2 + "CAP_FOWNER", // 3 + "CAP_FSETID", // 4 + "CAP_KILL", // 5 + "CAP_SETGID", // 6 + "CAP_SETUID", // 7 + "CAP_SETPCAP", // 8 + "CAP_LINUX_IMMUTABLE", // 9 + "CAP_NET_BIND_SERVICE", // 10 + "CAP_NET_BROADCAST", // 11 + "CAP_NET_ADMIN", // 12 + "CAP_NET_RAW", // 13 + "CAP_IPC_LOCK", // 14 + "CAP_IPC_OWNER", // 15 + "CAP_SYS_MODULE", // 16 + "CAP_SYS_RAWIO", // 17 + "CAP_SYS_CHROOT", // 18 + "CAP_SYS_PTRACE", // 19 + "CAP_SYS_PACCT", // 20 + "CAP_SYS_ADMIN", // 21 + "CAP_SYS_BOOT", // 22 + "CAP_SYS_NICE", // 23 + "CAP_SYS_RESOURCE", // 24 + "CAP_SYS_TIME", // 25 + "CAP_SYS_TTY_CONFIG", // 26 + "CAP_MKNOD", // 27 + "CAP_LEASE", // 28 + "CAP_AUDIT_WRITE", // 29 + "CAP_AUDIT_CONTROL", // 30 + "CAP_SETFCAP", // 31 + "CAP_MAC_OVERRIDE", // 32 + "CAP_MAC_ADMIN", // 33 + "CAP_SYSLOG", // 34 + "CAP_WAKE_ALARM", // 35 + "CAP_BLOCK_SUSPEND", // 36 + "CAP_AUDIT_READ", // 37 + "CAP_PERFMON", // 38 + "CAP_BPF", // 39 + "CAP_CHECKPOINT_RESTORE", // 40 + // The ECS spec allows for numerical string representation. + // The following capability values are not assigned as of Dec 28, 2023. + // If they are added in a future kernel, and this slice has not been + // updated, the numerical string will used. + "41", + "42", + "43", + "44", + "45", + "46", + "47", + "48", + "49", + "50", + "51", + "52", + "53", + "54", + "55", + "56", + "57", + "58", + "59", + "60", + "61", + "62", + "63", + } +) + +func mustReadBootID() string { + bootID, err := os.ReadFile("/proc/sys/kernel/random/boot_id") + if err != nil { + panic(fmt.Sprintf("could not read /proc/sys/kernel/random/boot_id: %v", err)) + } + + return strings.TrimRight(string(bootID), "\n") +} + +func mustReadPidNsInode() uint64 { + var ret uint64 + + pidNsInodeRaw, err := os.Readlink("/proc/self/ns/pid") + if err != nil { + panic(fmt.Sprintf("could not read /proc/self/ns/pid: %v", err)) + } + + if _, err = fmt.Sscanf(pidNsInodeRaw, "pid:[%d]", &ret); err != nil { + panic(fmt.Sprintf("could not parse contents of /proc/self/ns/pid (%s): %v", pidNsInodeRaw, err)) + } + + return ret +} + +func pidInfoFromProto(p types.PidInfo) types.PidInfo { + return types.PidInfo{ + StartTimeNs: p.StartTimeNs, + Tid: p.Tid, + Tgid: p.Tgid, + Vpid: p.Vpid, + Ppid: p.Ppid, + Pgid: p.Pgid, + Sid: p.Sid, + } +} + +func credInfoFromProto(p types.CredInfo) types.CredInfo { + return types.CredInfo{ + Ruid: p.Ruid, + Rgid: p.Rgid, + Euid: p.Euid, + Egid: p.Egid, + Suid: p.Suid, + Sgid: p.Sgid, + CapPermitted: p.CapPermitted, + CapEffective: p.CapEffective, + } +} + +func ttyTermiosFromProto(p types.TtyTermios) types.TtyTermios { + return types.TtyTermios{ + CIflag: p.CIflag, + COflag: p.COflag, + CLflag: p.CLflag, + CCflag: p.CCflag, + } +} + +func ttyWinsizeFromProto(p types.TtyWinsize) types.TtyWinsize { + return types.TtyWinsize{ + Rows: p.Rows, + Cols: p.Cols, + } +} + +func ttyDevFromProto(p types.TtyDev) types.TtyDev { + return types.TtyDev{ + Major: p.Major, + Minor: p.Minor, + Winsize: ttyWinsizeFromProto(p.Winsize), + Termios: ttyTermiosFromProto(p.Termios), + } +} + +type SimpleDB struct { + sync.RWMutex + logger *logp.Logger + processes map[uint32]Process + procfs procfs.Reader +} + +func NewSimpleDB(reader procfs.Reader, logger logp.Logger) *SimpleDB { + ret := &SimpleDB{ + logger: logp.NewLogger("processdb"), + processes: make(map[uint32]Process), + procfs: reader, + } + + return ret +} + +func (db *SimpleDB) calculateEntityIDv1(pid uint32, startTime time.Time) string { + return base64.StdEncoding.EncodeToString( + []byte( + fmt.Sprintf("%d__%s__%d__%d", + pidNsInode, + bootID, + uint64(pid), + uint64(startTime.Unix()), + ), + ), + ) +} + +// `path.Base` returns a '.' for empty strings, this just special cases that +// situation to return an empty string +func basename(pathStr string) string { + if pathStr == "" { + return "" + } + + return path.Base(pathStr) +} + +func (db *SimpleDB) InsertFork(fork types.ProcessForkEvent) error { + db.Lock() + defer db.Unlock() + + pid := fork.ChildPids.Tgid + ppid := fork.ParentPids.Tgid + if entry, ok := db.processes[ppid]; ok { + entry.Pids = pidInfoFromProto(fork.ChildPids) + entry.Creds = credInfoFromProto(fork.Creds) + db.processes[pid] = entry + } else { + db.processes[pid] = Process{ + Pids: pidInfoFromProto(fork.ChildPids), + Creds: credInfoFromProto(fork.Creds), + } + } + + return nil +} + +func (db *SimpleDB) insertProcess(process Process) { + pid := process.Pids.Tgid + db.processes[pid] = process +} + +func (db *SimpleDB) InsertExec(exec types.ProcessExecEvent) error { + db.Lock() + defer db.Unlock() + + proc := Process{ + Pids: pidInfoFromProto(exec.Pids), + Creds: credInfoFromProto(exec.Creds), + CTty: ttyDevFromProto(exec.CTty), + Argv: exec.Argv, + Cwd: exec.Cwd, + Env: exec.Env, + Filename: exec.Filename, + } + + db.processes[exec.Pids.Tgid] = proc + return nil +} + +func (db *SimpleDB) InsertSetsid(setsid types.ProcessSetsidEvent) error { + db.Lock() + defer db.Unlock() + + if entry, ok := db.processes[setsid.Pids.Tgid]; ok { + entry.Pids = pidInfoFromProto(setsid.Pids) + db.processes[setsid.Pids.Tgid] = entry + } else { + db.processes[setsid.Pids.Tgid] = Process{ + Pids: pidInfoFromProto(setsid.Pids), + } + } + + return nil +} + +func (db *SimpleDB) InsertExit(exit types.ProcessExitEvent) error { + db.Lock() + defer db.Unlock() + + pid := exit.Pids.Tgid + delete(db.processes, pid) + return nil +} + +// TODO: is this the correct definition? I looked in endpoint and I swear it looks too simple/generalized +func interactiveFromTty(tty types.TtyDev) bool { + return TtyUnknown != getTtyType(tty.Major, tty.Minor) +} + +func ecsCapsFromU64(capabilities uint64) []string { + var ecsCaps []string + if c := bits.OnesCount64(capabilities); c > 0 { + ecsCaps = make([]string, 0, c) + } + for bitnum := 0; bitnum < 64; bitnum++ { + if (capabilities & (1 << bitnum)) > 0 { + ecsCaps = append(ecsCaps, capNames[bitnum]) + } + } + return ecsCaps +} + +func fullProcessFromDBProcess(p Process) types.Process { + reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(p.Pids.StartTimeNs) + interactive := interactiveFromTty(p.CTty) + + ret := types.Process{ + PID: p.Pids.Tgid, + Start: timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime), + Name: basename(p.Filename), + Executable: p.Filename, + Args: p.Argv, + WorkingDirectory: p.Cwd, + Interactive: &interactive, + } + + euid := p.Creds.Euid + egid := p.Creds.Egid + ret.User.ID = strconv.FormatUint(uint64(euid), 10) + ret.Group.ID = strconv.FormatUint(uint64(egid), 10) + ret.Thread.Capabilities.Permitted = ecsCapsFromU64(p.Creds.CapPermitted) + ret.Thread.Capabilities.Effective = ecsCapsFromU64(p.Creds.CapEffective) + + return ret +} + +func fillParent(process *types.Process, parent Process) { + reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(parent.Pids.StartTimeNs) + + interactive := interactiveFromTty(parent.CTty) + euid := parent.Creds.Euid + egid := parent.Creds.Egid + process.Parent.PID = parent.Pids.Tgid + process.Parent.Start = timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime) + process.Parent.Name = basename(parent.Filename) + process.Parent.Executable = parent.Filename + process.Parent.Args = parent.Argv + process.Parent.WorkingDirectory = parent.Cwd + process.Parent.Interactive = &interactive + process.Parent.User.ID = strconv.FormatUint(uint64(euid), 10) + process.Parent.Group.ID = strconv.FormatUint(uint64(egid), 10) +} + +func fillGroupLeader(process *types.Process, groupLeader Process) { + reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(groupLeader.Pids.StartTimeNs) + + interactive := interactiveFromTty(groupLeader.CTty) + euid := groupLeader.Creds.Euid + egid := groupLeader.Creds.Egid + process.GroupLeader.PID = groupLeader.Pids.Tgid + process.GroupLeader.Start = timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime) + process.GroupLeader.Name = basename(groupLeader.Filename) + process.GroupLeader.Executable = groupLeader.Filename + process.GroupLeader.Args = groupLeader.Argv + process.GroupLeader.WorkingDirectory = groupLeader.Cwd + process.GroupLeader.Interactive = &interactive + process.GroupLeader.User.ID = strconv.FormatUint(uint64(euid), 10) + process.GroupLeader.Group.ID = strconv.FormatUint(uint64(egid), 10) +} + +func fillSessionLeader(process *types.Process, sessionLeader Process) { + reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(sessionLeader.Pids.StartTimeNs) + + interactive := interactiveFromTty(sessionLeader.CTty) + euid := sessionLeader.Creds.Euid + egid := sessionLeader.Creds.Egid + process.SessionLeader.PID = sessionLeader.Pids.Tgid + process.SessionLeader.Start = timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime) + process.SessionLeader.Name = basename(sessionLeader.Filename) + process.SessionLeader.Executable = sessionLeader.Filename + process.SessionLeader.Args = sessionLeader.Argv + process.SessionLeader.WorkingDirectory = sessionLeader.Cwd + process.SessionLeader.Interactive = &interactive + process.SessionLeader.User.ID = strconv.FormatUint(uint64(euid), 10) + process.SessionLeader.Group.ID = strconv.FormatUint(uint64(egid), 10) +} + +func (db *SimpleDB) setEntityID(process *types.Process) { + if process.PID != 0 && process.Start != nil { + process.EntityID = db.calculateEntityIDv1(process.PID, *process.Start) + } + + if process.Parent.PID != 0 && process.Parent.Start != nil { + process.Parent.EntityID = db.calculateEntityIDv1(process.Parent.PID, *process.Parent.Start) + } + + if process.GroupLeader.PID != 0 && process.GroupLeader.Start != nil { + process.GroupLeader.EntityID = db.calculateEntityIDv1(process.GroupLeader.PID, *process.GroupLeader.Start) + } + + if process.SessionLeader.PID != 0 && process.SessionLeader.Start != nil { + process.SessionLeader.EntityID = db.calculateEntityIDv1(process.SessionLeader.PID, *process.SessionLeader.Start) + } +} + +func setSameAsProcess(process *types.Process) { + if process.GroupLeader.PID != 0 && process.GroupLeader.Start != nil { + sameAsProcess := process.PID == process.GroupLeader.PID + process.GroupLeader.SameAsProcess = &sameAsProcess + } + + if process.SessionLeader.PID != 0 && process.SessionLeader.Start != nil { + sameAsProcess := process.PID == process.SessionLeader.PID + process.SessionLeader.SameAsProcess = &sameAsProcess + } +} + +func (db *SimpleDB) GetProcess(pid uint32) (types.Process, error) { + db.RLock() + defer db.RUnlock() + + process, ok := db.processes[pid] + if !ok { + return types.Process{}, errors.New("process not found") + } + + ret := fullProcessFromDBProcess(process) + + if parent, ok := db.processes[process.Pids.Ppid]; ok { + fillParent(&ret, parent) + } + + if groupLeader, ok := db.processes[process.Pids.Pgid]; ok { + fillGroupLeader(&ret, groupLeader) + } + + if sessionLeader, ok := db.processes[process.Pids.Sid]; ok { + fillSessionLeader(&ret, sessionLeader) + } + + db.setEntityID(&ret) + setSameAsProcess(&ret) + + return ret, nil +} + +func (db *SimpleDB) ScrapeProcfs() []uint32 { + db.Lock() + defer db.Unlock() + + procs, err := db.procfs.GetAllProcesses() + if err != nil { + db.logger.Errorf("failed to get processes from procfs: %v", err) + return make([]uint32, 0) + } + + // sorting the slice to make sure that parents, session leaders, group + // leaders come first in the queue + sort.Slice(procs, func(i, j int) bool { + return procs[i].Pids.Tgid == procs[j].Pids.Ppid || + procs[i].Pids.Tgid == procs[j].Pids.Sid || + procs[i].Pids.Tgid == procs[j].Pids.Pgid + }) + + pids := make([]uint32, 0) + for _, procInfo := range procs { + process := Process{ + Pids: pidInfoFromProto(procInfo.Pids), + Creds: credInfoFromProto(procInfo.Creds), + CTty: ttyDevFromProto(procInfo.CTty), + Argv: procInfo.Argv, + Cwd: procInfo.Cwd, + Env: procInfo.Env, + Filename: procInfo.Filename, + } + + db.insertProcess(process) + pids = append(pids, process.Pids.Tgid) + } + + return pids +} diff --git a/x-pack/auditbeat/processors/add_session_metadata/pkg/processdb/simple_test.go b/x-pack/auditbeat/processors/add_session_metadata/pkg/processdb/simple_test.go new file mode 100644 index 000000000000..9446c749b717 --- /dev/null +++ b/x-pack/auditbeat/processors/add_session_metadata/pkg/processdb/simple_test.go @@ -0,0 +1,55 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package processdb + +import ( + "testing" + + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/pkg/procfs" + "github.com/stretchr/testify/assert" + + "github.com/elastic/elastic-agent-libs/logp" + + "golang.org/x/sys/unix" +) + +var logger = logp.NewLogger("processdb") +var reader = procfs.NewMockReader() + +// glue function to fit the return type required by these tests +func newSimpleDBIntf(reader procfs.Reader) DB { + ret := NewSimpleDB(reader, *logger) + _ = ret.ScrapeProcfs() + return ret +} + +func TestCapsFromU64ToECS(t *testing.T) { + expected := []string{"CAP_CHOWN"} + assert.Equal(t, expected, ecsCapsFromU64(uint64(1<> 8) & 0xf) +} + +func MinorTty(ttyNr uint32) uint16 { + return uint16(((ttyNr & 0xfff00000) >> 20) | (ttyNr & 0xff)) +} + +// this interface exists so that we can inject a mock procfs reader for deterministic testing +type Reader interface { + GetProcess(pid uint32) (ProcessInfo, error) + GetAllProcesses() ([]ProcessInfo, error) +} + +type ProcfsReader struct { + logger logp.Logger +} + +func NewProcfsReader(logger logp.Logger) ProcfsReader { + return ProcfsReader{ + logger: logger, + } +} + +type Stat p.ProcStat + +type ProcessInfo struct { + Pids types.PidInfo + Creds types.CredInfo + CTty types.TtyDev + Argv []string + Cwd string + Env map[string]string + Filename string + CGroupPath string +} + +func credsFromProc(proc p.Proc) (types.CredInfo, error) { + status, err := proc.NewStatus() + if err != nil { + return types.CredInfo{}, err + } + + ruid, err := strconv.Atoi(status.UIDs[0]) + if err != nil { + return types.CredInfo{}, err + } + + euid, err := strconv.Atoi(status.UIDs[1]) + if err != nil { + return types.CredInfo{}, err + } + + suid, err := strconv.Atoi(status.UIDs[2]) + if err != nil { + return types.CredInfo{}, err + } + + rgid, err := strconv.Atoi(status.GIDs[0]) + if err != nil { + return types.CredInfo{}, err + } + + egid, err := strconv.Atoi(status.GIDs[1]) + if err != nil { + return types.CredInfo{}, err + } + + sgid, err := strconv.Atoi(status.GIDs[2]) + if err != nil { + return types.CredInfo{}, err + } + + // procfs library doesn't grab CapEff or CapPrm, make the direct syscall + hdr := unix.CapUserHeader{ + Version: unix.LINUX_CAPABILITY_VERSION_3, + Pid: int32(proc.PID), + } + var data [2]unix.CapUserData + err = unix.Capget(&hdr, &data[0]) + if err != nil { + return types.CredInfo{}, err + } + permitted := uint64(data[1].Permitted) << 32 + permitted += uint64(data[0].Permitted) + effective := uint64(data[1].Effective) << 32 + effective += uint64(data[0].Effective) + + return types.CredInfo{ + Ruid: uint32(ruid), + Euid: uint32(euid), + Suid: uint32(suid), + Rgid: uint32(rgid), + Egid: uint32(egid), + Sgid: uint32(sgid), + CapPermitted: permitted, + CapEffective: effective, + }, nil +} + +func (r ProcfsReader) getProcessInfo(proc p.Proc) (ProcessInfo, error) { + pid := uint32(proc.PID) + // All other info can be best effort, but failing to get pid info and + // start time is needed to register the process in the database + stat, err := proc.Stat() + if err != nil { + return ProcessInfo{}, fmt.Errorf("failed to read /proc/%d/stat: %v", pid, err) + } + + argv, err := proc.CmdLine() + if err != nil { + argv = []string{} + } + + exe, err := proc.Executable() + if err != nil { + if len(argv) > 0 { + r.logger.Debugf("pid %d: got executable from cmdline: %s", pid, argv[0]) + exe = argv[0] + } else { + r.logger.Debugf("pid %d: failed to get executable path: %v", pid, err) + exe = "" + } + } + + environ, err := r.getEnviron(pid) + if err != nil { + environ = nil + } + + cwd, err := proc.Cwd() + if err != nil { + cwd = "" + } + + creds, err := credsFromProc(proc) + if err != nil { + creds = types.CredInfo{} + } + + cGroupPath := "" + cgroups, err := proc.Cgroups() + if err == nil { + out: + // Find the cgroup path from the PID controller. + // NOTE: This does not support the unified hierarchy from cgroup v2, as bpf also does not currently support it. + // When support is added for unified hierarchies, it should be added in bpf and userspace at the same time. + // (Currently all supported cgroup v2 systems (GKE) are working as they send backwards compatible v1 hierarchies as well) + for _, cgroup := range cgroups { + for _, controller := range cgroup.Controllers { + if controller == "pids" { + cGroupPath = cgroup.Path + break out + } + } + } + } + + startTimeNs := timeutils.TicksToNs(stat.Starttime) + return ProcessInfo{ + Pids: types.PidInfo{ + StartTimeNs: startTimeNs, + Tid: pid, + Tgid: pid, + Ppid: uint32(stat.PPID), + Pgid: uint32(stat.PGRP), + Sid: uint32(stat.Session), + }, + Creds: creds, + CTty: types.TtyDev{ + Major: MajorTty(uint32(stat.TTY)), + Minor: MinorTty(uint32(stat.TTY)), + }, + Cwd: cwd, + Argv: argv, + Env: environ, + Filename: exe, + CGroupPath: cGroupPath, + }, nil +} + +func (r ProcfsReader) GetProcess(pid uint32) (ProcessInfo, error) { + proc, err := p.NewProc(int(pid)) + if err != nil { + return ProcessInfo{}, err + } + return r.getProcessInfo(proc) +} +// returns empty slice on error +func (r ProcfsReader) GetAllProcesses() ([]ProcessInfo, error) { + procs, err := p.AllProcs() + if err != nil { + return nil, err + } + + ret := make([]ProcessInfo, 0) + for _, proc := range procs { + process_info, err := r.getProcessInfo(proc) + if err != nil { + r.logger.Warnf("failed to read process info for %v", proc.PID) + } + ret = append(ret, process_info) + } + + return ret, nil +} + +func (r ProcfsReader) getEnviron(pid uint32) (map[string]string, error) { + proc, err := p.NewProc(int(pid)) + if err != nil { + return nil, err + } + + flatEnviron, err := proc.Environ() + if err != nil { + return nil, err + } + + ret := make(map[string]string) + for _, entry := range flatEnviron { + index := strings.Index(entry, "=") + if index == -1 { + continue + } + + ret[entry[0:index]] = entry[index:] + } + + return ret, nil +} diff --git a/x-pack/auditbeat/processors/add_session_metadata/pkg/timeutils/time.go b/x-pack/auditbeat/processors/add_session_metadata/pkg/timeutils/time.go new file mode 100644 index 000000000000..daa733349f54 --- /dev/null +++ b/x-pack/auditbeat/processors/add_session_metadata/pkg/timeutils/time.go @@ -0,0 +1,62 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package timeutils + +import ( + "fmt" + "time" + + p "github.com/prometheus/procfs" + "github.com/tklauser/go-sysconf" +) + +var ( + bootTime = mustGetBootTime() + ticksPerSecond = mustGetTicksPerSecond() +) + +func mustGetBootTime() time.Time { + fs, err := p.NewDefaultFS() + if err != nil { + panic(fmt.Sprintf("could not get procfs: %v", err)) + } + + stat, err := fs.Stat() + if err != nil { + panic(fmt.Sprintf("could not read /proc/stat: %v", err)) + } + return time.Unix(int64(stat.BootTime), 0) +} + +func mustGetTicksPerSecond() uint64 { + tps, err := sysconf.Sysconf(sysconf.SC_CLK_TCK) + if err != nil { + panic(fmt.Sprintf("sysconf(SC_CLK_TCK) failed: %v", err)) + } + return uint64(tps) +} + +func TicksToNs(ticks uint64) uint64 { + return ticks * uint64(time.Second.Nanoseconds()) / ticksPerSecond +} + +func TimeFromNsSinceBoot(ns uint64) *time.Time { + timestamp := bootTime.Add(time.Duration(ns)) + return ×tamp +} + +// When generating an `entity_id` in ECS we need to reduce the precision of a +// process's start time to that of procfs. Process start times can come from either +// BPF (high precision) or procfs (lower precision). We must reduce them all to the +// lowest common denominator such that entity ID's generated are always consistent. +// +// - Timestamps we get from the kernel are in nanosecond precision. +// - Timestamps we get from procfs are typically 1/100th second precision. We +// get this precision from `sysconf()` +// - We store timestamps as nanoseconds, but reduce the precision to 1/100th +// second +func ReduceTimestampPrecision(timeNs uint64) uint64 { + return timeNs - (timeNs % (uint64(time.Second.Nanoseconds()) / ticksPerSecond)) +} diff --git a/x-pack/auditbeat/processors/add_session_metadata/pkg/timeutils/time_test.go b/x-pack/auditbeat/processors/add_session_metadata/pkg/timeutils/time_test.go new file mode 100644 index 000000000000..94557e160274 --- /dev/null +++ b/x-pack/auditbeat/processors/add_session_metadata/pkg/timeutils/time_test.go @@ -0,0 +1,22 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package timeutils + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestReduceTimestampPrecision(t *testing.T) { + oneSecond := uint64(time.Second.Nanoseconds()) + result1 := ReduceTimestampPrecision(oneSecond) + require.Equal(t, oneSecond, result1) + + oneSecondWithDelay := oneSecond + 10 + result2 := ReduceTimestampPrecision(oneSecondWithDelay) + require.Equal(t, oneSecond, result2) +} diff --git a/x-pack/auditbeat/processors/add_session_metadata/provider/ebpf_provider/ebpf_provider.go b/x-pack/auditbeat/processors/add_session_metadata/provider/ebpf_provider/ebpf_provider.go new file mode 100644 index 000000000000..c83fcf37cda3 --- /dev/null +++ b/x-pack/auditbeat/processors/add_session_metadata/provider/ebpf_provider/ebpf_provider.go @@ -0,0 +1,159 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package ebpf_provider + +import ( + "context" + + "github.com/mohae/deepcopy" + + "github.com/elastic/beats/v7/libbeat/beat" + + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/pkg/processdb" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/provider" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/types" + + "github.com/elastic/elastic-agent-libs/logp" + + "github.com/elastic/ebpfevents" +) + +type prvdr struct { + ctx context.Context + logger logp.Logger + db processdb.DB +} + +func NewProvider(ctx context.Context, logger logp.Logger, db processdb.DB) (provider.Provider, error) { + p := prvdr{ + ctx: ctx, + logger: logger, + db: db, + } + + l, err := ebpfevents.NewLoader() + if err != nil { + p.logger.Errorf("loading ebpf: %w", err) + return prvdr{}, err + } + + events := make(chan ebpfevents.Event) + errors := make(chan error) + + go l.EventLoop(p.ctx, events, errors) + + go func(logger logp.Logger) { + for { + select { + case err := <-errors: + logger.Errorf("recv'd error: %w", err) + continue + case ev := <-events: + if err != nil { + logger.Errorf("marshal event: %w", err) + continue + } + switch ev.Type { + case ebpfevents.EventTypeProcessFork: + body, ok := ev.Body.(*ebpfevents.ProcessFork) + if !ok { + logger.Errorf("unexpected event body") + continue + } + pe := types.ProcessForkEvent{ + ParentPids: types.PidInfo{ + Tid: body.ParentPids.Tid, + Tgid: body.ParentPids.Tgid, + Ppid: body.ParentPids.Ppid, + Pgid: body.ParentPids.Pgid, + Sid: body.ParentPids.Sid, + StartTimeNs: body.ParentPids.StartTimeNs, + }, + ChildPids: types.PidInfo{ + Tid: body.ChildPids.Tid, + Tgid: body.ChildPids.Tgid, + Ppid: body.ChildPids.Ppid, + Pgid: body.ChildPids.Pgid, + Sid: body.ChildPids.Sid, + StartTimeNs: body.ChildPids.StartTimeNs, + }, + Creds: types.CredInfo{ + Ruid: body.Creds.Ruid, + Rgid: body.Creds.Rgid, + Euid: body.Creds.Euid, + Egid: body.Creds.Egid, + Suid: body.Creds.Suid, + Sgid: body.Creds.Sgid, + CapPermitted: body.Creds.CapPermitted, + CapEffective: body.Creds.CapEffective, + }, + } + p.db.InsertFork(pe) + case ebpfevents.EventTypeProcessExec: + body, ok := ev.Body.(*ebpfevents.ProcessExec) + if !ok { + logger.Errorf("unexpected event body") + continue + } + pe := types.ProcessExecEvent{ + Pids: types.PidInfo{ + Tid: body.Pids.Tid, + Tgid: body.Pids.Tgid, + Ppid: body.Pids.Ppid, + Pgid: body.Pids.Pgid, + Sid: body.Pids.Sid, + StartTimeNs: body.Pids.StartTimeNs, + }, + Creds: types.CredInfo{ + Ruid: body.Creds.Ruid, + Rgid: body.Creds.Rgid, + Euid: body.Creds.Euid, + Egid: body.Creds.Egid, + Suid: body.Creds.Suid, + Sgid: body.Creds.Sgid, + CapPermitted: body.Creds.CapPermitted, + CapEffective: body.Creds.CapEffective, + }, + CTty: types.TtyDev{ + Major: body.CTTY.Major, + Minor: body.CTTY.Minor, + }, + Cwd: body.Cwd, + Argv: deepcopy.Copy(body.Argv).([]string), + Env: deepcopy.Copy(body.Env).(map[string]string), + Filename: body.Filename, + } + p.db.InsertExec(pe) + case ebpfevents.EventTypeProcessExit: + body, ok := ev.Body.(*ebpfevents.ProcessExit) + if !ok { + logger.Errorf("unexpected event body") + continue + } + pe := types.ProcessExitEvent{ + Pids: types.PidInfo{ + Tid: body.Pids.Tid, + Tgid: body.Pids.Tgid, + Ppid: body.Pids.Ppid, + Pgid: body.Pids.Pgid, + Sid: body.Pids.Sid, + StartTimeNs: body.Pids.StartTimeNs, + }, + ExitCode: body.ExitCode, + } + p.db.InsertExit(pe) + } + continue + } + } + }(p.logger) + + return &p, nil +} + +func (s prvdr) UpdateDB(ev *beat.Event) error { + // no-op for ebpf, DB is updated from pushed ebpf events + return nil +} diff --git a/x-pack/auditbeat/processors/add_session_metadata/provider/provider.go b/x-pack/auditbeat/processors/add_session_metadata/provider/provider.go new file mode 100644 index 000000000000..c651c1fcffb4 --- /dev/null +++ b/x-pack/auditbeat/processors/add_session_metadata/provider/provider.go @@ -0,0 +1,13 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package provider + +import ( + "github.com/elastic/beats/v7/libbeat/beat" +) + +type Provider interface { + UpdateDB(*beat.Event) error +} diff --git a/x-pack/auditbeat/processors/add_session_metadata/types/events.go b/x-pack/auditbeat/processors/add_session_metadata/types/events.go new file mode 100644 index 000000000000..68dd53aa1d97 --- /dev/null +++ b/x-pack/auditbeat/processors/add_session_metadata/types/events.go @@ -0,0 +1,94 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package types + +//go:generate stringer -linecomment=true -type=Type,HookPoint,Field -output=gen_types_string.go + +type Type uint64 + +const ( + ProcessFork Type = iota + ProcessExec + ProcessExit + ProcessSetsid +) + +type ( + Field uint32 +) + +const ( + Cwd Field = iota + 1 + Argv + Env + Filename +) + +type PidInfo struct { + StartTimeNs uint64 + Tid uint32 + Tgid uint32 + Vpid uint32 + Ppid uint32 + Pgid uint32 + Sid uint32 +} + +type CredInfo struct { + Ruid uint32 + Rgid uint32 + Euid uint32 + Egid uint32 + Suid uint32 + Sgid uint32 + CapPermitted uint64 + CapEffective uint64 +} + +type TtyWinsize struct { + Rows uint16 + Cols uint16 +} + +type TtyTermios struct { + CIflag uint32 + COflag uint32 + CLflag uint32 + CCflag uint32 +} + +type TtyDev struct { + Minor uint16 + Major uint16 + Winsize TtyWinsize + Termios TtyTermios +} + +type ProcessForkEvent struct { + ParentPids PidInfo + ChildPids PidInfo + Creds CredInfo +} + +type ProcessExecEvent struct { + Pids PidInfo + Creds CredInfo + CTty TtyDev + + // varlen fields + Cwd string + Argv []string + Env map[string]string + Filename string +} + +type ProcessExitEvent struct { + Pids PidInfo + ExitCode int32 +} + +type ProcessSetsidEvent struct { + Pids PidInfo +} diff --git a/x-pack/auditbeat/processors/add_session_metadata/types/process.go b/x-pack/auditbeat/processors/add_session_metadata/types/process.go new file mode 100644 index 000000000000..cca5798c1deb --- /dev/null +++ b/x-pack/auditbeat/processors/add_session_metadata/types/process.go @@ -0,0 +1,367 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package types + +import ( + "time" + + "github.com/elastic/elastic-agent-libs/mapstr" +) + +// These fields contain information about a process. +// These fields can help you correlate metrics information with a process id/name from a log message. The `process.pid` often stays in the metric itself and is copied to the global field for correlation. +type Process struct { + // Unique identifier for the process. + // The implementation of this is specified by the data source, but some examples of what could be used here are a process-generated UUID, Sysmon Process GUIDs, or a hash of some uniquely identifying components of a process. + // Constructing a globally unique identifier is a common practice to mitigate PID reuse as well as to identify a specific process over time, across multiple monitored hosts. + EntityID string `json:"entity_id,omitempty"` + + // Absolute path to the process executable. + Executable string `json:"executable,omitempty"` + + // Process name. + // Sometimes called program name or similar. + Name string `json:"name,omitempty"` + + // The time the process started. + Start *time.Time `json:"start,omitempty"` + + // The time the process ended. + End *time.Time `json:"end,omitempty"` + + // The exit code of the process, if this is a termination event. + // The field should be absent if there is no exit code for the event (e.g. process start). + ExitCode *int64 `json:"exit_code,omitempty"` + + // Whether the process is connected to an interactive shell. + // Process interactivity is inferred from the processes file descriptors. If the character device for the controlling tty is the same as stdin and stderr for the process, the process is considered interactive. + // Note: A non-interactive process can belong to an interactive session and is simply one that does not have open file descriptors reading the controlling TTY on FD 0 (stdin) or writing to the controlling TTY on FD 2 (stderr). A backgrounded process is still considered interactive if stdin and stderr are connected to the controlling TTY. + Interactive *bool `json:"interactive,omitempty"` + + // The working directory of the process. + WorkingDirectory string `json:"working_directory,omitempty"` + + // The effective user (euid). + User struct { + // Unique identifier of the user. + ID string `json:"id,omitempty"` + + // Short name or login of the user. + Name string `json:"name,omitempty"` + } `json:"user,omitempty"` + + // The effective group (egid). + Group struct { + // Unique identifier for the group on the system/platform. + ID string `json:"id,omitempty"` + + // Name of the group. + Name string `json:"name,omitempty"` + } `json:"group,omitempty"` + + // Process id. + PID uint32 `json:"pid,omitempty"` + + Vpid uint32 `json:"vpid,omitempty"` + + // Array of process arguments, starting with the absolute path to the executable. + // May be filtered to protect sensitive information. + Args []string `json:"args,omitempty"` + + // An array of previous executions for the process, including the initial fork. Only executable and args are set. + Previous []struct { + // Absolute path to the process executable. + Executable string `json:"executable,omitempty"` + + // Array of process arguments, starting with the absolute path to the executable. + // May be filtered to protect sensitive information. + Args []string `json:"args,omitempty"` + } `json:"previous,omitempty"` + + Thread struct { + Capabilities struct { + Permitted []string `json:"permitted,omitempty"` + + Effective []string `json:"effective,omitempty"` + } `json:"capabilities,omitempty"` + } `json:"thread,omitempty"` + + // Information about the parent process. + Parent struct { + // Unique identifier for the process. + // The implementation of this is specified by the data source, but some examples of what could be used here are a process-generated UUID, Sysmon Process GUIDs, or a hash of some uniquely identifying components of a process. + // Constructing a globally unique identifier is a common practice to mitigate PID reuse as well as to identify a specific process over time, across multiple monitored hosts. + EntityID string `json:"entity_id,omitempty"` + + // Absolute path to the process executable. + Executable string `json:"executable,omitempty"` + + // Whether the process is connected to an interactive shell. + // Process interactivity is inferred from the processes file descriptors. If the character device for the controlling tty is the same as stdin and stderr for the process, the process is considered interactive. + // Note: A non-interactive process can belong to an interactive session and is simply one that does not have open file descriptors reading the controlling TTY on FD 0 (stdin) or writing to the controlling TTY on FD 2 (stderr). A backgrounded process is still considered interactive if stdin and stderr are connected to the controlling TTY. + Interactive *bool `json:"interactive,omitempty"` + + // Process name. + // Sometimes called program name or similar. + Name string `json:"name,omitempty"` + + // The time the process started. + Start *time.Time `json:"start,omitempty"` + + // The working directory of the process. + WorkingDirectory string `json:"working_directory,omitempty"` + + // The effective user (euid). + User struct { + // Unique identifier of the user. + ID string `json:"id,omitempty"` + + // Short name or login of the user. + Name string `json:"name,omitempty"` + } `json:"user,omitempty"` + + // The effective group (egid). + Group struct { + // Unique identifier for the group on the system/platform. + ID string `json:"id,omitempty"` + + // Name of the group. + Name string `json:"name,omitempty"` + } `json:"group,omitempty"` + + // Process id. + PID uint32 `json:"pid,omitempty"` + + // Array of process arguments, starting with the absolute path to the executable. + // May be filtered to protect sensitive information. + Args []string `json:"args,omitempty"` + + Thread struct { + Capabilities struct { + Permitted []string `json:"permitted,omitempty"` + + Effective []string `json:"effective,omitempty"` + } `json:"capabilities,omitempty"` + } `json:"thread,omitempty"` + } `json:"parent,omitempty"` + + // Information about the process group leader. In some cases this may be the same as the top level process. + GroupLeader struct { + // Unique identifier for the process. + // The implementation of this is specified by the data source, but some examples of what could be used here are a process-generated UUID, Sysmon Process GUIDs, or a hash of some uniquely identifying components of a process. + // Constructing a globally unique identifier is a common practice to mitigate PID reuse as well as to identify a specific process over time, across multiple monitored hosts. + EntityID string `json:"entity_id,omitempty"` + + // Absolute path to the process executable. + Executable string `json:"executable,omitempty"` + + // Whether the process is connected to an interactive shell. + // Process interactivity is inferred from the processes file descriptors. If the character device for the controlling tty is the same as stdin and stderr for the process, the process is considered interactive. + // Note: A non-interactive process can belong to an interactive session and is simply one that does not have open file descriptors reading the controlling TTY on FD 0 (stdin) or writing to the controlling TTY on FD 2 (stderr). A backgrounded process is still considered interactive if stdin and stderr are connected to the controlling TTY. + Interactive *bool `json:"interactive,omitempty"` + + // Process name. + // Sometimes called program name or similar. + Name string `json:"name,omitempty"` + + // The time the process started. + Start *time.Time `json:"start,omitempty"` + + // The working directory of the process. + WorkingDirectory string `json:"working_directory,omitempty"` + + // The effective user (euid). + User struct { + // Unique identifier of the user. + ID string `json:"id,omitempty"` + + // Short name or login of the user. + Name string `json:"name,omitempty"` + } `json:"user,omitempty"` + + // The effective group (egid). + Group struct { + // Unique identifier for the group on the system/platform. + ID string `json:"id,omitempty"` + + // Name of the group. + Name string `json:"name,omitempty"` + } `json:"group,omitempty"` + + // Process id. + PID uint32 `json:"pid,omitempty"` + + // Array of process arguments, starting with the absolute path to the executable. + // May be filtered to protect sensitive information. + Args []string `json:"args,omitempty"` + + // This boolean is used to identify if a leader process is the same as the top level process. + // For example, if `process.group_leader.same_as_process = true`, it means the process event in question is the leader of its process group. Details under `process.*` like `pid` would be the same under `process.group_leader.*` The same applies for both `process.session_leader` and `process.entry_leader`. + // This field exists to the benefit of EQL and other rule engines since it's not possible to compare equality between two fields in a single document. e.g `process.entity_id` = `process.group_leader.entity_id` (top level process is the process group leader) OR `process.entity_id` = `process.entry_leader.entity_id` (top level process is the entry session leader) + // Instead these rules could be written like: `process.group_leader.same_as_process: true` OR `process.entry_leader.same_as_process: true` + // Note: This field is only set on `process.entry_leader`, `process.session_leader` and `process.group_leader`. + SameAsProcess *bool `json:"same_as_process,omitempty"` + } `json:"group_leader,omitempty"` + + // Often the same as entry_leader. When it differs, it represents a session started within another session. e.g. using tmux + SessionLeader struct { + // Unique identifier for the process. + // The implementation of this is specified by the data source, but some examples of what could be used here are a process-generated UUID, Sysmon Process GUIDs, or a hash of some uniquely identifying components of a process. + // Constructing a globally unique identifier is a common practice to mitigate PID reuse as well as to identify a specific process over time, across multiple monitored hosts. + EntityID string `json:"entity_id,omitempty"` + + // Absolute path to the process executable. + Executable string `json:"executable,omitempty"` + + // Whether the process is connected to an interactive shell. + // Process interactivity is inferred from the processes file descriptors. If the character device for the controlling tty is the same as stdin and stderr for the process, the process is considered interactive. + // Note: A non-interactive process can belong to an interactive session and is simply one that does not have open file descriptors reading the controlling TTY on FD 0 (stdin) or writing to the controlling TTY on FD 2 (stderr). A backgrounded process is still considered interactive if stdin and stderr are connected to the controlling TTY. + Interactive *bool `json:"interactive,omitempty"` + + // Process name. + // Sometimes called program name or similar. + Name string `json:"name,omitempty"` + + // The time the process started. + Start *time.Time `json:"start,omitempty"` + + // The working directory of the process. + WorkingDirectory string `json:"working_directory,omitempty"` + + // The effective user (euid). + User struct { + // Unique identifier of the user. + ID string `json:"id,omitempty"` + + // Short name or login of the user. + Name string `json:"name,omitempty"` + } `json:"user,omitempty"` + + // The effective group (egid). + Group struct { + // Unique identifier for the group on the system/platform. + ID string `json:"id,omitempty"` + + // Name of the group. + Name string `json:"name,omitempty"` + } `json:"group,omitempty"` + + // Process id. + PID uint32 `json:"pid,omitempty"` + + // Array of process arguments, starting with the absolute path to the executable. + // May be filtered to protect sensitive information. + Args []string `json:"args,omitempty"` + + // This boolean is used to identify if a leader process is the same as the top level process. + // For example, if `process.group_leader.same_as_process = true`, it means the process event in question is the leader of its process group. Details under `process.*` like `pid` would be the same under `process.group_leader.*` The same applies for both `process.session_leader` and `process.entry_leader`. + // This field exists to the benefit of EQL and other rule engines since it's not possible to compare equality between two fields in a single document. e.g `process.entity_id` = `process.group_leader.entity_id` (top level process is the process group leader) OR `process.entity_id` = `process.entry_leader.entity_id` (top level process is the entry session leader) + // Instead these rules could be written like: `process.group_leader.same_as_process: true` OR `process.entry_leader.same_as_process: true` + // Note: This field is only set on `process.entry_leader`, `process.session_leader` and `process.group_leader`. + SameAsProcess *bool `json:"same_as_process,omitempty"` + } `json:"session_leader,omitempty"` +} + +func (p *Process) ToMap() mapstr.M { + process := mapstr.M{ + "entity_id": p.EntityID, + "executable": p.Executable, + "name": p.Name, + "exit_code": p.ExitCode, + "interactive": p.Interactive, + "working_directory": p.WorkingDirectory, + "user": mapstr.M{ + "id": p.User.ID, + "name": p.User.Name, + }, + "group": mapstr.M{ + "id": p.Group.ID, + "name": p.Group.Name, + }, + "pid": p.PID, + "vpid": p.Vpid, + "args": p.Args, + "thread": mapstr.M{ + "capabilities": mapstr.M{ + "permitted": p.Thread.Capabilities.Permitted, + "effective": p.Thread.Capabilities.Effective, + }, + }, + "parent": mapstr.M{ + "entity_id": p.Parent.EntityID, + "executable": p.Parent.Executable, + "name": p.Parent.Name, + "interactive": p.Parent.Interactive, + "working_directory": p.Parent.WorkingDirectory, + "user": mapstr.M{ + "id": p.Parent.User.ID, + "name": p.Parent.User.Name, + }, + "group": mapstr.M{ + "id": p.Parent.Group.ID, + "name": p.Parent.Group.Name, + }, + "pid": p.Parent.PID, + "args": p.Parent.Args, + "thread": mapstr.M{ + "capabilities": mapstr.M{ + "permitted": p.Parent.Thread.Capabilities.Permitted, + "effective": p.Parent.Thread.Capabilities.Effective, + }, + }, + }, + "group_leader": mapstr.M{ + "entity_id": p.GroupLeader.EntityID, + "executable": p.GroupLeader.Executable, + "name": p.GroupLeader.Name, + "interactive": p.GroupLeader.Interactive, + "working_directory": p.GroupLeader.WorkingDirectory, + "user": mapstr.M{ + "id": p.GroupLeader.User.ID, + "name": p.GroupLeader.User.Name, + }, + "group": mapstr.M{ + "id": p.GroupLeader.Group.ID, + "name": p.GroupLeader.Group.Name, + }, + "pid": p.GroupLeader.PID, + "args": p.GroupLeader.Args, + "same_as_process": p.GroupLeader.SameAsProcess, + }, + "session_leader": mapstr.M{ + "entity_id": p.SessionLeader.EntityID, + "executable": p.SessionLeader.Executable, + "name": p.SessionLeader.Name, + "interactive": p.SessionLeader.Interactive, + "working_directory": p.SessionLeader.WorkingDirectory, + "user": mapstr.M{ + "id": p.SessionLeader.User.ID, + "name": p.SessionLeader.User.Name, + }, + "group": mapstr.M{ + "id": p.SessionLeader.Group.ID, + "name": p.SessionLeader.Group.Name, + }, + "pid": p.SessionLeader.PID, + "args": p.SessionLeader.Args, + "same_as_process": p.SessionLeader.SameAsProcess, + }, + } + + // nil timestamps will cause a panic within the publisher, only add the mapping if it exists + if p.Start != nil { + process.Put("start", p.Start) + } + if p.Parent.Start != nil { + process.Put("parent.start", p.Parent.Start) + } + if p.GroupLeader.Start != nil { + process.Put("group_leader.start", p.GroupLeader.Start) + } + if p.SessionLeader.Start != nil { + process.Put("session_leader.start", p.SessionLeader.Start) + } + return process +} From d5a01bfa3805a6a564f1790e44270473a08419c0 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Thu, 11 Jan 2024 17:54:11 -0800 Subject: [PATCH 02/32] Add unit tests for add_session_metadata --- .../add_session_metadata.go | 1 - .../add_session_metadata_test.go | 211 ++++++++++++++++++ 2 files changed, 211 insertions(+), 1 deletion(-) create mode 100644 x-pack/auditbeat/processors/add_session_metadata/add_session_metadata_test.go diff --git a/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go b/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go index a412662cc6f5..4d0ab93112a7 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go +++ b/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go @@ -108,7 +108,6 @@ func (p *addSessionMetadata) enrich(ev *beat.Event) (*beat.Event, error) { if err != nil { return nil, err } - pid, err := pidToUInt32(pidIf) if err != nil { return nil, fmt.Errorf("cannot parse pid field '%s': %w", p.config.PidField, err) diff --git a/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata_test.go b/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata_test.go new file mode 100644 index 000000000000..d476d558048f --- /dev/null +++ b/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata_test.go @@ -0,0 +1,211 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package add_session_metadata + +import ( + "fmt" + "testing" + + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/pkg/processdb" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/pkg/procfs" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/types" + "github.com/stretchr/testify/assert" + + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +var ( + enrichTests = []struct{ + testName string + mockProcesses []types.ProcessExecEvent + config Config + input beat.Event + expected beat.Event + expect_error bool + }{ + { + testName: "Enrich Process", + config: Config{ + ReplaceFields: false, + PidField: "process.pid", + }, + mockProcesses: []types.ProcessExecEvent{ + { + Pids: types.PidInfo{ + Tid: uint32(100), + Tgid: uint32(100), + Ppid: uint32(50), + Pgid: uint32(100), + Sid: uint32(40), + }, + Cwd: "/", + Filename: "/bin/ls", + }, + { + Pids: types.PidInfo{ + Tid: uint32(50), + Tgid: uint32(50), + Ppid: uint32(40), + Sid: uint32(40), + }, + }, + { + Pids: types.PidInfo{ + Tid: uint32(40), + Tgid: uint32(40), + Ppid: uint32(1), + Sid: uint32(1), + }, + }, + }, + input: beat.Event{ + Fields: mapstr.M{ + "process": mapstr.M{ + "pid": uint32(100), + }, + }, + }, + expected: beat.Event{ + Fields: mapstr.M{ + "process": mapstr.M{ + "executable": "/bin/ls", + "working_directory": "/", + "pid": uint32(100), + "parent": mapstr.M{ + "pid": uint32(50), + }, + "session_leader": mapstr.M{ + "pid": uint32(40), + }, + "group_leader": mapstr.M{ + "pid": uint32(100), + }, + }, + }, + }, + expect_error: false, + }, + { + testName: "No Pid Field in Event", + config: Config{ + ReplaceFields: false, + PidField: "process.pid", + }, + input: beat.Event{ + Fields: mapstr.M{ + "process": mapstr.M{ + "executable": "ls", + "working_directory": "/", + "parent": mapstr.M{ + "pid": uint32(100), + }, + }, + }, + }, + expect_error: true, + }, + { + testName: "Pid Not Number", + config: Config{ + ReplaceFields: false, + PidField: "process.pid", + }, + input: beat.Event{ + Fields: mapstr.M{ + "process": mapstr.M{ + "pid": "xyz", + "executable": "ls", + "working_directory": "/", + "parent": mapstr.M{ + "pid": uint32(50), + }, + }, + }, + }, + expect_error: true, + }, + { + testName: "PID not in DB", + config: Config{ + ReplaceFields: false, + PidField: "process.pid", + }, + input: beat.Event{ + Fields: mapstr.M{ + "process": mapstr.M{ + "pid": "100", + "executable": "ls", + "working_directory": "/", + "parent": mapstr.M{ + "pid": uint32(100), + }, + }, + }, + }, + expect_error: true, + }, + } + + logger = logp.NewLogger("add_session_metadata_test") +) + +func TestEnrich(t *testing.T) { + for _, tt := range enrichTests { + reader := procfs.NewMockReader() + db := processdb.NewSimpleDB(reader, *logger) + + for _, ev := range tt.mockProcesses { + err := db.InsertExec(ev) + assert.NoError(t, err, "%s: inserting exec to db: %w", tt.testName, err) + } + s := addSessionMetadata{ + logger: logger, + db: db, + config: tt.config, + } + + actual, err := s.enrich(&tt.input) + if tt.expect_error { + assert.Error(t, err, "%s: error unexpectedly nil", tt.testName) + } else { + assert.Nil(t, err, "%s: enrich error: %w", tt.testName, err) + assert.NotNil(t, actual, "%s: returned nil event", tt.testName) + //Validate output + eq, msg := compareMapstr(tt.expected.Fields, actual.Fields) + assert.True(t, eq, "%s: actual does not match expected: %s\n\nactual: \"%v\"\n\nexpected: \"%v\"", tt.testName, msg, actual, tt.expected) + } + } +} + +// compareMapstr will compare that all fields in `a` have equal value to the fields in `b`. +// Note: Only fields that exist in `a` are compared; if `b` has additional fields, they do not affect the comparison +func compareMapstr(a mapstr.M, b mapstr.M) (equal bool, msg string) { + equal = false + msg = "" + + aFlat := a.Flatten() + bFlat := b.Flatten() + + for _, key := range *a.FlattenKeys() { + valA, err := aFlat.GetValue(key) + if err == nil { + // FlattenKeys returns inner and leaf nodes, we only need to consider leaf nodes + // GetValue will return error when attempting to read inner nodes; these keys are ignored + valB, err := bFlat.GetValue(key) + if err != nil { + msg = fmt.Sprintf("%s not found in mapstr b", key) + return + } + if valA != valB { + msg = fmt.Sprintf("mismatch in key %s: \"%v\" \"%v\"", key, valA, valB) + return + } + } + } + equal = true + return +} From 7cec455ab8f40b95efbd35bab2ebc99027f31c24 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Mon, 15 Jan 2024 13:28:17 -0800 Subject: [PATCH 03/32] Calculate process entry leader Calculate and append entry leader information to enriched processes. --- .../add_session_metadata.go | 13 +- .../add_session_metadata/pkg/processdb/db.go | 51 +++++ .../pkg/processdb/simple.go | 175 ++++++++++++++++++ .../pkg/processdb/simple_test.go | 76 ++++++++ .../add_session_metadata/types/process.go | 89 +++++++++ 5 files changed, 396 insertions(+), 8 deletions(-) diff --git a/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go b/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go index 4d0ab93112a7..dc14d76d1be4 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go +++ b/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go @@ -13,8 +13,6 @@ import ( "github.com/elastic/elastic-agent-libs/mapstr" - "github.com/google/uuid" - "github.com/elastic/elastic-agent-libs/monitoring" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/pkg/processdb" @@ -177,19 +175,18 @@ func (p *addSessionMetadata) replaceFields(ev *beat.Event) error { if err != nil { return err } - if syscall == "execve" { + switch syscall { + case "execveat": + fallthrough + case "execve": ev.Fields.Put("event.action", []string{"exec", "fork"}) ev.Fields.Put("event.type", []string{"start"}) - } - // process end - if syscall == "exit_group" { + case "exit_group": ev.Fields.Put("event.action", []string{"end"}) ev.Fields.Put("event.type", []string{"end"}) ev.Fields.Put("process.end", time.Now()) } - - ev.Fields.Put("event.id", uuid.NewString()) } return nil } diff --git a/x-pack/auditbeat/processors/add_session_metadata/pkg/processdb/db.go b/x-pack/auditbeat/processors/add_session_metadata/pkg/processdb/db.go index 15857d262b62..30206e0b2ea5 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/pkg/processdb/db.go +++ b/x-pack/auditbeat/processors/add_session_metadata/pkg/processdb/db.go @@ -5,6 +5,8 @@ package processdb import ( + "strings" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/types" ) @@ -14,6 +16,7 @@ type DB interface { InsertSetsid(setsid types.ProcessSetsidEvent) error InsertExit(exit types.ProcessExitEvent) error GetProcess(pid uint32) (types.Process, error) + GetEntryType(pid uint32) (EntryType, error) ScrapeProcfs() []uint32 } @@ -26,6 +29,36 @@ const ( TtyConsole ) +type EntryType string + +const ( + Init EntryType = "init" + Sshd EntryType = "sshd" + Ssm EntryType = "ssm" + Container EntryType = "container" + Terminal EntryType = "terminal" + EntryConsole EntryType = "console" + EntryUnknown EntryType = "unknown" +) + +var containerRuntimes = [...]string{ + "containerd-shim", + "runc", + "conmon", +} + +// "filtered" executables are executables that relate to internal +// implementation details of entry mechanisms. The set of circumstances under +// which they can become an entry leader are reduced compared to other binaries +// (see implementation and unit tests). +var filteredExecutables = [...]string{ + "runc", + "containerd-shim", + "calico-node", + "check-status", + "conmon", +} + const ( ptsMinMajor = 136 ptsMaxMajor = 143 @@ -34,6 +67,24 @@ const ( ttyMaxMinor = 255 ) +func stringStartsWithEntryInList(str string, list []string) bool { + for _, entry := range list { + if strings.HasPrefix(str, entry) { + return true + } + } + + return false +} + +func isContainerRuntime(executable string) bool { + return stringStartsWithEntryInList(executable, containerRuntimes[:]) +} + +func isFilteredExecutable(executable string) bool { + return stringStartsWithEntryInList(executable, filteredExecutables[:]) +} + func getTtyType(major uint16, minor uint16) TtyType { if major >= ptsMinMajor && major <= ptsMaxMajor { return Pts diff --git a/x-pack/auditbeat/processors/add_session_metadata/pkg/processdb/simple.go b/x-pack/auditbeat/processors/add_session_metadata/pkg/processdb/simple.go index 43c56d014449..acf9de5bf7ff 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/pkg/processdb/simple.go +++ b/x-pack/auditbeat/processors/add_session_metadata/pkg/processdb/simple.go @@ -188,6 +188,8 @@ type SimpleDB struct { sync.RWMutex logger *logp.Logger processes map[uint32]Process + entryLeaders map[uint32]EntryType + entryLeaderRelationships map[uint32]uint32 procfs procfs.Reader } @@ -195,6 +197,8 @@ func NewSimpleDB(reader procfs.Reader, logger logp.Logger) *SimpleDB { ret := &SimpleDB{ logger: logp.NewLogger("processdb"), processes: make(map[uint32]Process), + entryLeaders: make(map[uint32]EntryType), + entryLeaderRelationships: make(map[uint32]uint32), procfs: reader, } @@ -234,6 +238,9 @@ func (db *SimpleDB) InsertFork(fork types.ProcessForkEvent) error { entry.Pids = pidInfoFromProto(fork.ChildPids) entry.Creds = credInfoFromProto(fork.Creds) db.processes[pid] = entry + if entryPid, ok := db.entryLeaderRelationships[ppid]; ok { + db.entryLeaderRelationships[pid] = entryPid + } } else { db.processes[pid] = Process{ Pids: pidInfoFromProto(fork.ChildPids), @@ -247,6 +254,13 @@ func (db *SimpleDB) InsertFork(fork types.ProcessForkEvent) error { func (db *SimpleDB) insertProcess(process Process) { pid := process.Pids.Tgid db.processes[pid] = process + entryLeaderPid := db.evaluateEntryLeader(process) + if entryLeaderPid != nil { + db.entryLeaderRelationships[pid] = *entryLeaderPid + db.logger.Debugf("%v name: %s, entry_leader: %d, entry_type: %s", process.Pids, process.Filename, *entryLeaderPid, string(db.entryLeaders[*entryLeaderPid])) + } else { + db.logger.Debugf("%v name: %s, NO ENTRY LEADER", process.Pids, process.Filename) + } } func (db *SimpleDB) InsertExec(exec types.ProcessExecEvent) error { @@ -264,6 +278,116 @@ func (db *SimpleDB) InsertExec(exec types.ProcessExecEvent) error { } db.processes[exec.Pids.Tgid] = proc + entryLeaderPid := db.evaluateEntryLeader(proc) + if entryLeaderPid != nil { + db.entryLeaderRelationships[exec.Pids.Tgid] = *entryLeaderPid + } + + return nil +} + +func (db *SimpleDB) createEntryLeader(pid uint32, entryType EntryType) { + db.entryLeaders[pid] = entryType + db.logger.Debugf("created entry leader %d: %s, name: %s", pid, string(entryType), db.processes[pid].Filename) +} + +// pid returned is a pointer type because its possible for no +func (db *SimpleDB) evaluateEntryLeader(p Process) *uint32 { + pid := p.Pids.Tgid + + // init never has an entry leader or meta type + if p.Pids.Tgid == 1 { + db.logger.Debugf("entry_eval %d: process is init, no entry type", p.Pids.Tgid) + return nil + } + + // kernel threads also never have an entry leader or meta type kthreadd + // (always pid 2) is the parent of all kernel threads, by filtering pid == + // 2 || ppid == 2, we get rid of all of them + if p.Pids.Tgid == 2 || p.Pids.Ppid == 2 { + db.logger.Debugf("entry_eval %d: kernel threads never an entry type (parent is pid 2)", p.Pids.Tgid) + return nil + } + + // could be an entry leader + if p.Pids.Tgid == p.Pids.Sid { + ttyType := getTtyType(p.CTty.Major, p.CTty.Minor) + + procBasename := basename(p.Filename) + if ttyType == Tty { + db.createEntryLeader(pid, Terminal) + db.logger.Debugf("entry_eval %d: entry type is terminal", p.Pids.Tgid) + return &pid + } else if ttyType == TtyConsole && procBasename == "login" { + db.createEntryLeader(pid, EntryConsole) + db.logger.Debugf("entry_eval %d: entry type is console", p.Pids.Tgid) + return &pid + } else if p.Pids.Ppid == 1 { + db.createEntryLeader(pid, Init) + db.logger.Debugf("entry_eval %d: entry type is init", p.Pids.Tgid) + return &pid + } else if !isFilteredExecutable(procBasename) { + if parent, ok := db.processes[p.Pids.Ppid]; ok { + parentBasename := basename(parent.Filename) + if ttyType == Pts && parentBasename == "ssm-session-worker" { + db.createEntryLeader(pid, Ssm) + db.logger.Debugf("entry_eval %d: entry type is ssm", p.Pids.Tgid) + return &pid + } else if parentBasename == "sshd" && procBasename != "sshd" { + // TODO: get ip from env vars + db.createEntryLeader(pid, Sshd) + db.logger.Debugf("entry_eval %d: entry type is sshd", p.Pids.Tgid) + return &pid + } else if isContainerRuntime(parentBasename) { + db.createEntryLeader(pid, Container) + db.logger.Debugf("entry_eval %d: entry type is container", p.Pids.Tgid) + return &pid + } + } + } else { + db.logger.Debugf("entry_eval %d: is a filtered executable: %s", p.Pids.Tgid, procBasename) + } + } + + // if not a session leader or was not determined to be an entry leader, get + // it via parent, session leader, group leader (in that order) + relations := []struct { + pid uint32 + name string + }{ + { + pid: p.Pids.Ppid, + name: "parent", + }, + { + pid: p.Pids.Sid, + name: "session_leader", + }, + { + pid: p.Pids.Pgid, + name: "group_leader", + }, + } + + for _, relation := range relations { + if entry, ok := db.entryLeaderRelationships[relation.pid]; ok { + entryType := db.entryLeaders[entry] + db.logger.Debugf("entry_eval %d: got entry_leader: %d (%s), from relative: %d (%s)", p.Pids.Tgid, entry, string(entryType), relation.pid, relation.name) + return &entry + } else { + db.logger.Debugf("entry_eval %d: failed to find relative: %d (%s)", p.Pids.Tgid, relation.pid, relation.name) + } + } + + // if it's a session leader, then make it its own entry leader with unknown + // entry type + if p.Pids.Tgid == p.Pids.Sid { + db.createEntryLeader(pid, EntryUnknown) + db.logger.Debugf("entry_eval %d: this is a session leader and no relative has an entry leader. entry type is unknown", p.Pids.Tgid) + return &pid + } + + db.logger.Debugf("entry_eval %d: this is not a session leader and no relative has an entry leader, entry_leader will be unset", p.Pids.Tgid) return nil } @@ -289,6 +413,8 @@ func (db *SimpleDB) InsertExit(exit types.ProcessExitEvent) error { pid := exit.Pids.Tgid delete(db.processes, pid) + delete(db.entryLeaders, pid) + delete(db.entryLeaderRelationships, pid) return nil } @@ -385,6 +511,25 @@ func fillSessionLeader(process *types.Process, sessionLeader Process) { process.SessionLeader.Group.ID = strconv.FormatUint(uint64(egid), 10) } +func fillEntryLeader(process *types.Process, entryType EntryType, entryLeader Process) { + reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(entryLeader.Pids.StartTimeNs) + + interactive := interactiveFromTty(entryLeader.CTty) + euid := entryLeader.Creds.Euid + egid := entryLeader.Creds.Egid + process.EntryLeader.PID = entryLeader.Pids.Tgid + process.EntryLeader.Start = timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime) + process.EntryLeader.Name = basename(entryLeader.Filename) + process.EntryLeader.Executable = entryLeader.Filename + process.EntryLeader.Args = entryLeader.Argv + process.EntryLeader.WorkingDirectory = entryLeader.Cwd + process.EntryLeader.Interactive = &interactive + process.EntryLeader.User.ID = strconv.FormatUint(uint64(euid), 10) + process.EntryLeader.Group.ID = strconv.FormatUint(uint64(egid), 10) + + process.EntryLeader.EntryMeta.Type = string(entryType) +} + func (db *SimpleDB) setEntityID(process *types.Process) { if process.PID != 0 && process.Start != nil { process.EntityID = db.calculateEntityIDv1(process.PID, *process.Start) @@ -401,6 +546,10 @@ func (db *SimpleDB) setEntityID(process *types.Process) { if process.SessionLeader.PID != 0 && process.SessionLeader.Start != nil { process.SessionLeader.EntityID = db.calculateEntityIDv1(process.SessionLeader.PID, *process.SessionLeader.Start) } + + if process.EntryLeader.PID != 0 && process.EntryLeader.Start != nil { + process.EntryLeader.EntityID = db.calculateEntityIDv1(process.EntryLeader.PID, *process.EntryLeader.Start) + } } func setSameAsProcess(process *types.Process) { @@ -413,6 +562,11 @@ func setSameAsProcess(process *types.Process) { sameAsProcess := process.PID == process.SessionLeader.PID process.SessionLeader.SameAsProcess = &sameAsProcess } + + if process.EntryLeader.PID != 0 && process.EntryLeader.Start != nil { + sameAsProcess := process.PID == process.EntryLeader.PID + process.EntryLeader.SameAsProcess = &sameAsProcess + } } func (db *SimpleDB) GetProcess(pid uint32) (types.Process, error) { @@ -438,12 +592,33 @@ func (db *SimpleDB) GetProcess(pid uint32) (types.Process, error) { fillSessionLeader(&ret, sessionLeader) } + if entryLeaderPid, foundEntryLeaderPid := db.entryLeaderRelationships[process.Pids.Tgid]; foundEntryLeaderPid { + if entryLeader, foundEntryLeader := db.processes[entryLeaderPid]; foundEntryLeader { + // if there is an entry leader then there is a matching member in the entryLeaders table + fillEntryLeader(&ret, db.entryLeaders[entryLeaderPid], entryLeader) + } else { + db.logger.Errorf("failed to find entry leader entry %d for %d (%s)", entryLeaderPid, pid, db.processes[pid].Filename) + } + } else { + db.logger.Errorf("failed to find entry leader for %d (%s)", pid, db.processes[pid].Filename) + } + db.setEntityID(&ret) setSameAsProcess(&ret) return ret, nil } +func (db *SimpleDB) GetEntryType(pid uint32) (EntryType, error) { + db.RLock() + defer db.RUnlock() + + if entryType, ok := db.entryLeaders[pid]; ok { + return entryType, nil + } + return EntryUnknown, nil +} + func (db *SimpleDB) ScrapeProcfs() []uint32 { db.Lock() defer db.Unlock() diff --git a/x-pack/auditbeat/processors/add_session_metadata/pkg/processdb/simple_test.go b/x-pack/auditbeat/processors/add_session_metadata/pkg/processdb/simple_test.go index 9446c749b717..a5df2cd869c6 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/pkg/processdb/simple_test.go +++ b/x-pack/auditbeat/processors/add_session_metadata/pkg/processdb/simple_test.go @@ -25,6 +25,82 @@ func newSimpleDBIntf(reader procfs.Reader) DB { return ret } +func TestSimpleSingleProcessSessionLeaderEntryTypeTerminal(t *testing.T) { + testSingleProcessSessionLeaderEntryTypeTerminal(newSimpleDBIntf)(t) +} + +func TestSimpleSingleProcessSessionLeaderLoginProcess(t *testing.T) { + testSingleProcessSessionLeaderLoginProcess(newSimpleDBIntf)(t) +} + +func TestSimpleSingleProcessSessionLeaderChildOfInit(t *testing.T) { + testSingleProcessSessionLeaderChildOfInit(newSimpleDBIntf)(t) +} + +func TestSimpleSingleProcessSessionLeaderChildOfSsmSessionWorker(t *testing.T) { + testSingleProcessSessionLeaderChildOfSsmSessionWorker(newSimpleDBIntf)(t) +} + +func TestSimpleSingleProcessSessionLeaderChildOfSshd(t *testing.T) { + testSingleProcessSessionLeaderChildOfSshd(newSimpleDBIntf)(t) +} + +func TestSimpleSingleProcessSessionLeaderChildOfContainerdShim(t *testing.T) { + testSingleProcessSessionLeaderChildOfContainerdShim(newSimpleDBIntf)(t) +} + +func TestSimpleSingleProcessSessionLeaderOfRunc(t *testing.T) { + testSingleProcessSessionLeaderChildOfRunc(newSimpleDBIntf)(t) +} + +func TestSimpleSingleProcessEmptyProcess(t *testing.T) { + testSingleProcessEmptyProcess(newSimpleDBIntf)(t) +} + +func TestSimpleSingleProcessOverwriteOldEntryLeader(t *testing.T) { + testSingleProcessOverwriteOldEntryLeader(newSimpleDBIntf)(t) +} + +func TestSimpleInitSshdBashLs(t *testing.T) { + testInitSshdBashLs(newSimpleDBIntf)(t) +} + +func TestSimpleInitSshdSshdBashLs(t *testing.T) { + testInitSshdSshdBashLs(newSimpleDBIntf)(t) +} + +func TestSimpleInitSshdSshdSshdBashLs(t *testing.T) { + testInitSshdSshdSshdBashLs(newSimpleDBIntf)(t) +} + +func TestSimpleInitContainerdContainerdShim(t *testing.T) { + testInitContainerdContainerdShim(newSimpleDBIntf)(t) +} + +func TestSimpleInitContainerdShimBashContainerdShimIsReparentedToInit(t *testing.T) { + testInitContainerdShimBashContainerdShimIsReparentedToInit(newSimpleDBIntf)(t) +} + +func TestSimpleInitContainerdShimPauseContainerdShimIsReparentedToInit(t *testing.T) { + testInitContainerdShimPauseContainerdShimIsReparentedToInit(newSimpleDBIntf)(t) +} + +func TestSimpleInitSshdBashLsAndGrepGrepOnlyHasGroupLeader(t *testing.T) { + testInitSshdBashLsAndGrepGrepOnlyHasGroupLeader(newSimpleDBIntf)(t) +} + +func TestSimpleInitSshdBashLsAndGrepGrepOnlyHasSessionLeader(t *testing.T) { + testInitSshdBashLsAndGrepGrepOnlyHasSessionLeader(newSimpleDBIntf)(t) +} + +func TestSimpleGrepInIsolation(t *testing.T) { + testGrepInIsolation(newSimpleDBIntf)(t) +} + +func TestSimpleKernelThreads(t *testing.T) { + testKernelThreads(newSimpleDBIntf)(t) +} + func TestCapsFromU64ToECS(t *testing.T) { expected := []string{"CAP_CHOWN"} assert.Equal(t, expected, ecsCapsFromU64(uint64(1< Date: Mon, 15 Jan 2024 16:15:56 -0800 Subject: [PATCH 04/32] Add entry leader tests --- .../add_session_metadata.go | 8 +- .../processors/add_session_metadata/config.go | 4 +- .../pkg/processdb/entry_leader_test.go | 1247 +++++++++++++++++ 3 files changed, 1254 insertions(+), 5 deletions(-) create mode 100644 x-pack/auditbeat/processors/add_session_metadata/pkg/processdb/entry_leader_test.go diff --git a/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go b/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go index dc14d76d1be4..0d88b55d4ca5 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go +++ b/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go @@ -157,10 +157,12 @@ func pidToUInt32(value interface{}) (pid uint32, err error) { return pid, nil } -//replaceFields replaces event fields with values suitable for session view +// replaceFields replaces event fields with values suitable user with the session viewer in Kibana +// The current version of session view in Kibana expects different values than what are used by auditbeat +// for some fields. This function converts these field to have values that will work with session view. +// +// This function is temporary, and can be removed when Kibana is updated to work with the auditbeat field values. func (p *addSessionMetadata) replaceFields(ev *beat.Event) error { - // This will only work with auditd data, but this is entire func is temporary - // It should be removed when session view can work with the original fields. kind, err := ev.Fields.GetValue("event.kind") if err != nil { return err diff --git a/x-pack/auditbeat/processors/add_session_metadata/config.go b/x-pack/auditbeat/processors/add_session_metadata/config.go index 71673ec8deb6..8a8cfc75e6d7 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/config.go +++ b/x-pack/auditbeat/processors/add_session_metadata/config.go @@ -4,7 +4,7 @@ package add_session_metadata -// Config for add_host_metadata processor. +// Config for add_session_metadata processor. type Config struct { Backend string `config:"backend"` ReplaceFields bool `config:"replace_fields"` @@ -14,7 +14,7 @@ type Config struct { func defaultConfig() Config { return Config{ Backend: "ebpf", - ReplaceFields: true, + ReplaceFields: false, PidField: "process.pid", } } diff --git a/x-pack/auditbeat/processors/add_session_metadata/pkg/processdb/entry_leader_test.go b/x-pack/auditbeat/processors/add_session_metadata/pkg/processdb/entry_leader_test.go new file mode 100644 index 000000000000..ec2f4358f070 --- /dev/null +++ b/x-pack/auditbeat/processors/add_session_metadata/pkg/processdb/entry_leader_test.go @@ -0,0 +1,1247 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package processdb + +import ( + "path" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/pkg/procfs" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/types" + // "github.com/elastic/elastic-agent-libs/logp" +) + +type ( + createDBFn func(procfs.Reader) DB + testFn func(*testing.T) +) + +const ( + containerdShimPath = "/bin/containerd-shim-runc-v2" + containerdPath = "/bin/containerd" + sshdPath = "/usr/bin/sshd" + lsPath = "/usr/bin/ls" + bashPath = "/usr/bin/bash" + grepPath = "/usr/bin/grep" +) + +// Entry evaluation tests +// +// The entry leader isn't an entirely rigorous conceptual framework but that +// shortcoming is outweighted by the large and immediate value it provides. +// +// The idea is to assign two pieces of data to each process, the "entry meta" +// and "entry leader", the former of which describes how the user or system +// that was ultimately responsible for executing this process got into to the +// box (e.g. ssh, ssm, kubectl exec) and the latter of which describes the +// process associated with the user or system's initial entry into the "box" +// (be it a container, VM or otherwise). +// +// Generally speaking, the first session leader in a process lineage of an +// interactive session is an entry leader having an entry meta type depending +// on its lineage. For example, in the following process tree, "bash" is an +// entry leader with entry meta type "sshd": +// +// systemd (pid 1 sid 1) +// \___ sshd (pid 100 sid 100) +// \___ bash (pid 1000 sid 1000) +// \___ vim (pid 1001 sid 1000) +// +// Further entry meta types exist for ssm, container runtimes, serial consoles +// and other ways to get into a "box" (be it a container or actual machine). +// The entry meta type "init" is assigned to system processes created by the +// init service (e.g. rsyslogd, sshd). +// +// As should probably be apparent, the code to assign an entry meta type to a +// process is essentially a large amount of conditional logic with a ton of +// edge cases. It's something we "bolt on" to the linux process model, and thus +// finicky and highly subject to bugs. +// +// Thankfully, writing unit tests for entry leader evaluation is rather +// straightforward as it's basically a pure function that requires no external +// infrastructure to test (just create a mock process event with your desired +// fields set and pass it in). +// +// These tests should effectively serve as the spec for how we assign entry +// leaders. When further entry meta types or cases are added, tests should be + +func requireProcess(t *testing.T, db DB, pid uint32, processPath string) { + process, err := db.GetProcess(pid) + require.Nil(t, err) + require.Equal(t, pid, process.PID) + require.Equal(t, processPath, process.Executable) + if processPath == "" { + require.Equal(t, "", process.Name) + } else { + require.Equal(t, path.Base(processPath), process.Name) + } +} + +func requireParent(t *testing.T, db DB, pid uint32, ppid uint32) { + process, err := db.GetProcess(pid) + require.Nil(t, err) + require.Equal(t, ppid, process.Parent.PID) +} + +func requireParentUnset(t *testing.T, process types.Process) { + require.Equal(t, "", process.Parent.EntityID) + require.Equal(t, uint32(0), process.Parent.PID) + require.Nil(t, process.Parent.Start) +} + +func requireSessionLeader(t *testing.T, db DB, pid uint32, sid uint32) { + process, err := db.GetProcess(pid) + require.Nil(t, err) + require.Equal(t, sid, process.SessionLeader.PID) + require.NotNil(t, process.SessionLeader.SameAsProcess) + require.Equal(t, pid == sid, *process.SessionLeader.SameAsProcess) +} + +func requireSessionLeaderUnset(t *testing.T, process types.Process) { + require.Equal(t, "", process.SessionLeader.EntityID) + require.Equal(t, uint32(0), process.SessionLeader.PID) + require.Nil(t, process.SessionLeader.Start) +} + +func requireGroupLeader(t *testing.T, db DB, pid uint32, pgid uint32) { + process, err := db.GetProcess(pid) + require.Nil(t, err) + require.Equal(t, pgid, process.GroupLeader.PID) + require.NotNil(t, process.GroupLeader.SameAsProcess) + require.Equal(t, pid == pgid, *process.GroupLeader.SameAsProcess) +} + +func requireEntryLeader(t *testing.T, db DB, pid uint32, entryPid uint32, expectedEntryType EntryType) { + process, err := db.GetProcess(pid) + require.Nil(t, err) + require.Equal(t, entryPid, process.EntryLeader.PID) + require.NotNil(t, process.EntryLeader.SameAsProcess) + require.Equal(t, pid == entryPid, *process.EntryLeader.SameAsProcess) + + entryType, err := db.GetEntryType(entryPid) + require.Nil(t, err) + require.Equal(t, expectedEntryType, entryType) +} + +func requireEntryLeaderUnset(t *testing.T, process types.Process) { + require.Equal(t, "", process.EntryLeader.EntityID) + require.Equal(t, uint32(0), process.EntryLeader.PID) + require.Nil(t, process.EntryLeader.Start) +} + +// tries to construct fork event from what's in the db +func insertForkAndExec(t *testing.T, db DB, exec types.ProcessExecEvent) { + var fork types.ProcessForkEvent + fork.ChildPids = exec.Pids + parent, err := db.GetProcess(exec.Pids.Ppid) + if err != nil { + fork.ParentPids = exec.Pids + fork.ParentPids.Tgid = exec.Pids.Ppid + fork.ParentPids.Ppid = 0 + fork.ParentPids.Pgid = 0 + + fork.ChildPids.Pgid = exec.Pids.Ppid + + // if the exec makes itself a session and the parent is no where to be + // found we'll make the parent its own session + if exec.Pids.Tgid == exec.Pids.Sid { + fork.ParentPids.Sid = exec.Pids.Ppid + } + } else { + fork.ParentPids.Tgid = parent.PID + fork.ParentPids.Ppid = parent.Parent.PID + fork.ParentPids.Sid = parent.SessionLeader.PID + + // keep group leader the same for now + fork.ParentPids.Pgid = exec.Pids.Pgid + } + + if fork.ParentPids.Tgid != 0 { + err = db.InsertFork(fork) + require.Nil(t, err) + } + + err = db.InsertExec(exec) + require.Nil(t, err) +} + +var systemdPath = "/sbin/systemd" + +func populateProcfsWithInit(reader *procfs.MockReader) { + reader.AddEntry(1, procfs.ProcessInfo{ + Pids: types.PidInfo{ + Tid: 1, + Tgid: 1, + Pgid: 0, + Sid: 1, + }, + Filename: systemdPath, + }) +} + +func testSingleProcessSessionLeaderEntryTypeTerminal(createDB createDBFn) testFn { + return func(t *testing.T) { + reader := procfs.NewMockReader() + db := createDB(reader) + + pid := uint32(1234) + procPath := "/bin/noproc" + err := db.InsertExec(types.ProcessExecEvent{ + Filename: procPath, + Pids: types.PidInfo{ + Tgid: pid, + Sid: pid, + }, + CTty: types.TtyDev{ + Major: 4, + Minor: 64, + }, + }) + require.Nil(t, err) + + requireProcess(t, db, 1234, procPath) + requireEntryLeader(t, db, 1234, 1234, Terminal) + } +} + +func testSingleProcessSessionLeaderLoginProcess(createDB createDBFn) testFn { + return func(t *testing.T) { + reader := procfs.NewMockReader() + db := createDB(reader) + + pid := uint32(1234) + loginPath := "/bin/login" + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: loginPath, + Pids: types.PidInfo{ + Tgid: pid, + Sid: pid, + }, + CTty: types.TtyDev{ + Major: 4, + Minor: 62, + }, + }) + + process, err := db.GetProcess(1234) + require.Nil(t, err) + requireParentUnset(t, process) + + requireProcess(t, db, pid, "/bin/login") + requireSessionLeader(t, db, pid, pid) + requireEntryLeader(t, db, pid, pid, EntryConsole) + } +} + +func testSingleProcessSessionLeaderChildOfInit(createDB createDBFn) testFn { + return func(t *testing.T) { + reader := procfs.NewMockReader() + db := createDB(reader) + + pid := uint32(100) + rsyslogdPath := "/bin/rsyslogd" + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: rsyslogdPath, + Pids: types.PidInfo{ + Tgid: pid, + Sid: pid, + Ppid: 1, + }, + CTty: types.TtyDev{ + Major: 136, + Minor: 62, + }, + }) + + process, err := db.GetProcess(1234) + require.NotNil(t, err) + requireParentUnset(t, process) + + requireProcess(t, db, pid, rsyslogdPath) + requireSessionLeader(t, db, pid, pid) + requireEntryLeader(t, db, pid, pid, Init) + } +} + +func testSingleProcessSessionLeaderChildOfSsmSessionWorker(createDB createDBFn) testFn { + return func(t *testing.T) { + reader := procfs.NewMockReader() + db := createDB(reader) + + ssmPid := uint32(999) + bashPid := uint32(1000) + ssmPath := "/usr/bin/ssm-session-worker" + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: ssmPath, + Pids: types.PidInfo{ + Tgid: ssmPid, + }, + }) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: bashPath, + Pids: types.PidInfo{ + Tgid: bashPid, + Sid: bashPid, + Ppid: ssmPid, + }, + CTty: types.TtyDev{ + Major: 136, + Minor: 62, + }, + }) + + requireProcess(t, db, bashPid, bashPath) + requireParent(t, db, bashPid, ssmPid) + requireSessionLeader(t, db, bashPid, bashPid) + requireEntryLeader(t, db, bashPid, bashPid, Ssm) + } +} + +func testSingleProcessSessionLeaderChildOfSshd(createDB createDBFn) testFn { + return func(t *testing.T) { + reader := procfs.NewMockReader() + db := createDB(reader) + + sshdPid := uint32(999) + bashPid := uint32(1000) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: sshdPath, + Pids: types.PidInfo{ + Tgid: sshdPid, + }, + }) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: bashPath, + Pids: types.PidInfo{ + Tgid: bashPid, + Sid: bashPid, + Ppid: sshdPid, + }, + CTty: types.TtyDev{ + Major: 136, + Minor: 62, + }, + }) + + requireProcess(t, db, bashPid, bashPath) + requireParent(t, db, bashPid, sshdPid) + requireSessionLeader(t, db, bashPid, bashPid) + requireEntryLeader(t, db, bashPid, bashPid, Sshd) + } +} + +func testSingleProcessSessionLeaderChildOfContainerdShim(createDB createDBFn) testFn { + return func(t *testing.T) { + reader := procfs.NewMockReader() + db := createDB(reader) + + containerdShimPid := uint32(999) + bashPid := uint32(1000) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: containerdShimPath, + Pids: types.PidInfo{ + Tgid: containerdShimPid, + }, + }) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: bashPath, + Pids: types.PidInfo{ + Tgid: bashPid, + Sid: bashPid, + Ppid: containerdShimPid, + }, + CTty: types.TtyDev{ + Major: 136, + Minor: 62, + }, + }) + + requireProcess(t, db, bashPid, bashPath) + requireParent(t, db, bashPid, containerdShimPid) + requireSessionLeader(t, db, bashPid, bashPid) + requireEntryLeader(t, db, bashPid, bashPid, Container) + } +} + +func testSingleProcessSessionLeaderChildOfRunc(createDB createDBFn) testFn { + return func(t *testing.T) { + reader := procfs.NewMockReader() + db := createDB(reader) + + runcPid := uint32(999) + bashPid := uint32(1000) + runcPath := "/bin/runc" + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: runcPath, + Pids: types.PidInfo{ + Tgid: runcPid, + }, + }) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: bashPath, + Pids: types.PidInfo{ + Tgid: bashPid, + Sid: bashPid, + Ppid: runcPid, + }, + CTty: types.TtyDev{ + Major: 136, + Minor: 62, + }, + }) + + requireProcess(t, db, bashPid, bashPath) + requireParent(t, db, bashPid, runcPid) + requireSessionLeader(t, db, bashPid, bashPid) + requireEntryLeader(t, db, bashPid, bashPid, Container) + } +} + +func testSingleProcessEmptyProcess(createDB createDBFn) testFn { + return func(t *testing.T) { + reader := procfs.NewMockReader() + db := createDB(reader) + + // No information in proc at all, entry type should be "unknown" + // and entry leader pid should be unset (since pid is not set) + pid := uint32(1000) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: bashPath, + Pids: types.PidInfo{ + Tgid: pid, + Sid: pid, + }, + CTty: types.TtyDev{ + Major: 136, + Minor: 62, + }, + }) + + process, err := db.GetProcess(pid) + require.Nil(t, err) + requireParentUnset(t, process) + + requireProcess(t, db, pid, bashPath) + requireSessionLeader(t, db, pid, pid) + requireEntryLeader(t, db, pid, pid, EntryUnknown) + } +} + +// Entry evaluation code should overwrite an old EntryLeaderPid and +// EntryLeaderEntryMetaType +func testSingleProcessOverwriteOldEntryLeader(createDB createDBFn) testFn { + return func(t *testing.T) { + reader := procfs.NewMockReader() + db := createDB(reader) + + ssmPid := uint32(999) + bashPid := uint32(1000) + ssmPath := "/usr/bin/ssm-session-worker" + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: ssmPath, + Pids: types.PidInfo{ + Tgid: ssmPid, + Sid: ssmPid, + Ppid: 1, + }, + }) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: bashPath, + Pids: types.PidInfo{ + Tgid: bashPid, + Sid: ssmPid, + Ppid: ssmPid, + }, + CTty: types.TtyDev{ + Major: 136, + Minor: 62, + }, + }) + + // bash is not a session leader so it shouldn't be an entry leader. Its + // entry leader should be ssm, which is an init entry leader + requireProcess(t, db, bashPid, bashPath) + requireParent(t, db, bashPid, ssmPid) + requireSessionLeader(t, db, bashPid, ssmPid) + requireEntryLeader(t, db, bashPid, ssmPid, Init) + + // skiping setsid event and assuming the pids will be updated in this exec + err := db.InsertExec(types.ProcessExecEvent{ + Filename: bashPath, + Pids: types.PidInfo{ + Tgid: bashPid, + Sid: bashPid, + Ppid: ssmPid, + }, + CTty: types.TtyDev{ + Major: 136, + Minor: 62, + }, + }) + require.Nil(t, err) + + requireProcess(t, db, bashPid, bashPath) + requireParent(t, db, bashPid, ssmPid) + requireSessionLeader(t, db, bashPid, bashPid) + requireEntryLeader(t, db, bashPid, bashPid, Ssm) + } +} + +// / (pid, sid, entry meta, entry leader) +// +// systemd (1, 1, none, none) +// +// \___ sshd (100, 100, "init", 100) +// \___ bash (1000, 1000, "sshd", 1000) +// \___ ls (1001, 1000, "sshd", 1000) +// +// This is unrealistic, sshd usually forks a bunch of sshd children before +// exec'ing bash (see subsequent tests) but is theoretically possible and +// thus something we should handle. +func testInitSshdBashLs(createDB createDBFn) testFn { + return func(t *testing.T) { + reader := procfs.NewMockReader() + populateProcfsWithInit(reader) + db := createDB(reader) + + sshdPid := uint32(100) + bashPid := uint32(1000) + lsPid := uint32(1001) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: sshdPath, + Pids: types.PidInfo{ + Tgid: sshdPid, + Sid: sshdPid, + Ppid: 1, + }, + }) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: bashPath, + Pids: types.PidInfo{ + Tgid: bashPid, + Sid: bashPid, + Ppid: sshdPid, + Pgid: bashPid, + }, + }) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: lsPath, + Pids: types.PidInfo{ + Tgid: lsPid, + Sid: bashPid, + Ppid: bashPid, + Pgid: lsPid, + }, + }) + + // systemd + systemd, err := db.GetProcess(1) + require.Nil(t, err) + requireParentUnset(t, systemd) + requireEntryLeaderUnset(t, systemd) + + requireProcess(t, db, 1, systemdPath) + requireSessionLeader(t, db, 1, 1) + + // sshd + requireProcess(t, db, sshdPid, sshdPath) + requireParent(t, db, sshdPid, 1) + requireSessionLeader(t, db, sshdPid, sshdPid) + requireEntryLeader(t, db, sshdPid, sshdPid, Init) + + // bash + requireProcess(t, db, bashPid, bashPath) + requireParent(t, db, bashPid, sshdPid) + requireSessionLeader(t, db, bashPid, bashPid) + requireEntryLeader(t, db, bashPid, bashPid, Sshd) + requireGroupLeader(t, db, bashPid, bashPid) + + // ls + requireProcess(t, db, lsPid, lsPath) + requireParent(t, db, lsPid, bashPid) + requireSessionLeader(t, db, lsPid, bashPid) + requireEntryLeader(t, db, lsPid, bashPid, Sshd) + requireGroupLeader(t, db, lsPid, lsPid) + } +} + +// / (pid, sid, entry meta, entry leader) +// +// systemd (1, 1, none, none) +// +// \___ sshd (100, 100, "init", 100) +// \___ sshd (101, 101, "init", 100) +// \___ bash (1000, 1000, "sshd", 1000) +// \___ ls (1001, 1000, "sshd", 1000) +// +// sshd will usually fork a bunch of sshd children before invoking a shell +// usually 2 if it's a root shell, or 3 if it's a non-root shell. All +// "intermediate" sshd's should have entry meta "init" and an entry leader +// pid of the topmost sshd. +func testInitSshdSshdBashLs(createDB createDBFn) testFn { + return func(t *testing.T) { + reader := procfs.NewMockReader() + populateProcfsWithInit(reader) + db := createDB(reader) + + sshd0Pid := uint32(100) + sshd1Pid := uint32(101) + bashPid := uint32(1000) + lsPid := uint32(1001) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: sshdPath, + Pids: types.PidInfo{ + Tgid: sshd0Pid, + Sid: sshd0Pid, + Ppid: 1, + }, + }) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: sshdPath, + Pids: types.PidInfo{ + Tgid: sshd1Pid, + Sid: sshd1Pid, + Ppid: sshd0Pid, + }, + }) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: bashPath, + Pids: types.PidInfo{ + Tgid: bashPid, + Sid: bashPid, + Ppid: sshd1Pid, + Pgid: bashPid, + }, + }) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: lsPath, + Pids: types.PidInfo{ + Tgid: lsPid, + Sid: bashPid, + Ppid: bashPid, + Pgid: lsPid, + }, + }) + + // systemd + systemd, err := db.GetProcess(1) + require.Nil(t, err) + requireParentUnset(t, systemd) + requireEntryLeaderUnset(t, systemd) + + requireProcess(t, db, 1, systemdPath) + requireSessionLeader(t, db, 1, 1) + + // sshd0 + requireProcess(t, db, sshd0Pid, sshdPath) + requireParent(t, db, sshd0Pid, 1) + requireSessionLeader(t, db, sshd0Pid, sshd0Pid) + requireEntryLeader(t, db, sshd0Pid, sshd0Pid, Init) + + // sshd1 + requireProcess(t, db, sshd1Pid, sshdPath) + requireParent(t, db, sshd1Pid, sshd0Pid) + requireSessionLeader(t, db, sshd1Pid, sshd1Pid) + requireEntryLeader(t, db, sshd1Pid, sshd0Pid, Init) + + // bash + requireProcess(t, db, bashPid, bashPath) + requireParent(t, db, bashPid, sshd1Pid) + requireSessionLeader(t, db, bashPid, bashPid) + requireEntryLeader(t, db, bashPid, bashPid, Sshd) + + // ls + requireProcess(t, db, lsPid, lsPath) + requireParent(t, db, lsPid, bashPid) + requireSessionLeader(t, db, lsPid, bashPid) + requireEntryLeader(t, db, lsPid, bashPid, Sshd) + } +} + +// / (pid, sid, entry meta, entry leader) +// systemd (1, 1, none, none) +// +// \___ sshd (100, 100, "init", 100) +// \___ sshd (101, 101, "init", 100) +// \___ sshd (102, 101, "init", 100) +// \___ bash (1000, 1000, "sshd", 1000) +// \___ ls (1001, 1000, "sshd", 1000) +func testInitSshdSshdSshdBashLs(createDB createDBFn) testFn { + return func(t *testing.T) { + reader := procfs.NewMockReader() + populateProcfsWithInit(reader) + db := createDB(reader) + + sshd0Pid := uint32(100) + sshd1Pid := uint32(101) + sshd2Pid := uint32(102) + bashPid := uint32(1000) + lsPid := uint32(1001) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: sshdPath, + Pids: types.PidInfo{ + Tgid: sshd0Pid, + Sid: sshd0Pid, + Ppid: 1, + }, + }) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: sshdPath, + Pids: types.PidInfo{ + Tgid: sshd1Pid, + Sid: sshd1Pid, + Ppid: sshd0Pid, + }, + }) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: sshdPath, + Pids: types.PidInfo{ + Tgid: sshd2Pid, + Sid: sshd1Pid, + Ppid: sshd1Pid, + }, + }) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: bashPath, + Pids: types.PidInfo{ + Tgid: bashPid, + Sid: bashPid, + Ppid: sshd2Pid, + Pgid: bashPid, + }, + }) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: lsPath, + Pids: types.PidInfo{ + Tgid: lsPid, + Sid: bashPid, + Ppid: bashPid, + Pgid: lsPid, + }, + }) + + // systemd + systemd, err := db.GetProcess(1) + require.Nil(t, err) + requireParentUnset(t, systemd) + requireEntryLeaderUnset(t, systemd) + + requireProcess(t, db, 1, systemdPath) + requireSessionLeader(t, db, 1, 1) + + // sshd0 + requireProcess(t, db, sshd0Pid, sshdPath) + requireParent(t, db, sshd0Pid, 1) + requireSessionLeader(t, db, sshd0Pid, sshd0Pid) + requireEntryLeader(t, db, sshd0Pid, sshd0Pid, Init) + + // sshd1 + requireProcess(t, db, sshd1Pid, sshdPath) + requireParent(t, db, sshd1Pid, sshd0Pid) + requireSessionLeader(t, db, sshd1Pid, sshd1Pid) + requireEntryLeader(t, db, sshd1Pid, sshd0Pid, Init) + + // sshd2 + requireProcess(t, db, sshd2Pid, sshdPath) + requireParent(t, db, sshd2Pid, sshd1Pid) + requireSessionLeader(t, db, sshd2Pid, sshd1Pid) + requireEntryLeader(t, db, sshd2Pid, sshd0Pid, Init) + + // bash + requireProcess(t, db, bashPid, bashPath) + requireParent(t, db, bashPid, sshd2Pid) + requireSessionLeader(t, db, bashPid, bashPid) + requireEntryLeader(t, db, bashPid, bashPid, Sshd) + + // ls + requireProcess(t, db, lsPid, lsPath) + requireParent(t, db, lsPid, bashPid) + requireSessionLeader(t, db, lsPid, bashPid) + requireEntryLeader(t, db, lsPid, bashPid, Sshd) + } +} + +// / (pid, sid, entry meta, entry leader) +// +// systemd +// +// \___ containerd (100, 100, "init", 100) +// \___ containerd-shim-runc-v2 (1000, 100, "init", 100) +// +// containerd-shim-runc-v2 will reparent itself to init just prior to +// executing the containerized process. +func testInitContainerdContainerdShim(createDB createDBFn) testFn { + return func(t *testing.T) { + reader := procfs.NewMockReader() + populateProcfsWithInit(reader) + db := createDB(reader) + + containerdPid := uint32(100) + containerdShimPid := uint32(1000) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: containerdPath, + Pids: types.PidInfo{ + Tgid: containerdPid, + Sid: containerdPid, + Ppid: 1, + }, + }) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: containerdShimPath, + Pids: types.PidInfo{ + Tgid: containerdShimPid, + Sid: containerdPid, + Ppid: containerdPid, + }, + }) + + // containerd + requireProcess(t, db, containerdPid, containerdPath) + requireParent(t, db, containerdPid, 1) + requireSessionLeader(t, db, containerdPid, containerdPid) + requireEntryLeader(t, db, containerdPid, containerdPid, Init) + + // containerd-shim-runc-v2 + requireProcess(t, db, containerdShimPid, containerdShimPath) + requireParent(t, db, containerdShimPid, containerdPid) + requireSessionLeader(t, db, containerdShimPid, containerdPid) + requireEntryLeader(t, db, containerdShimPid, containerdPid, Init) + } +} + +// / (pid, sid, entry meta, entry leader) +// +// systemd +// +// \___ containerd (100, 100, "init", 100) +// | +// \___ containerd-shim-runc-v2 (1000, 100, "init", 100) +// \___ bash (1001, 1001, "container", 1000) +// +// Note that containerd originally forks and exec's +// containerd-shim-runc-v2, which then forks such that it is reparented to +// init. +func testInitContainerdShimBashContainerdShimIsReparentedToInit(createDB createDBFn) testFn { + return func(t *testing.T) { + reader := procfs.NewMockReader() + populateProcfsWithInit(reader) + db := createDB(reader) + + containerdPid := uint32(100) + containerdShimPid := uint32(1000) + bashPid := uint32(1001) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: containerdPath, + Pids: types.PidInfo{ + Tgid: containerdPid, + Sid: containerdPid, + Ppid: 1, + }, + }) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: containerdShimPath, + Pids: types.PidInfo{ + Tgid: containerdShimPid, + Sid: containerdPid, + Ppid: 1, + }, + }) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: bashPath, + Pids: types.PidInfo{ + Tgid: bashPid, + Sid: bashPid, + Ppid: containerdShimPid, + }, + }) + + // containerd + requireProcess(t, db, containerdPid, containerdPath) + requireParent(t, db, containerdPid, 1) + requireSessionLeader(t, db, containerdPid, containerdPid) + requireEntryLeader(t, db, containerdPid, containerdPid, Init) + + // containerd-shim-runc-v2 + requireProcess(t, db, containerdShimPid, containerdShimPath) + requireParent(t, db, containerdShimPid, 1) + requireSessionLeader(t, db, containerdShimPid, containerdPid) + requireEntryLeader(t, db, containerdShimPid, containerdPid, Init) + + // bash + requireProcess(t, db, bashPid, bashPath) + requireParent(t, db, bashPid, containerdShimPid) + requireSessionLeader(t, db, bashPid, bashPid) + requireEntryLeader(t, db, bashPid, bashPid, Container) + } +} + +// / (pid, sid, entry meta, entry leader) +// +// systemd +// +// \___ containerd (100, 100, "init", 100) +// | +// \___ containerd-shim-runc-v2 (1000, 100, "init", 100) +// \___ pause (1001, 1001, "container", 1001) +// +// The pause binary is a Kubernetes internal binary that is exec'd in a +// container by the container runtime. It is responsible for holding +// open the pod sandbox while other containers start and stop +func testInitContainerdShimPauseContainerdShimIsReparentedToInit(createDB createDBFn) testFn { + return func(t *testing.T) { + reader := procfs.NewMockReader() + populateProcfsWithInit(reader) + db := createDB(reader) + + containerdPid := uint32(100) + containerdShimPid := uint32(1000) + pausePid := uint32(1001) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: containerdPath, + Pids: types.PidInfo{ + Tgid: containerdPid, + Sid: containerdPid, + Ppid: 1, + }, + }) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: containerdShimPath, + Pids: types.PidInfo{ + Tgid: containerdShimPid, + Sid: containerdPid, + Ppid: 1, + }, + }) + + pausePath := "/usr/bin/pause" + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: pausePath, + Pids: types.PidInfo{ + Tgid: pausePid, + Sid: pausePid, + Ppid: containerdShimPid, + }, + }) + + // containerd + requireProcess(t, db, containerdPid, containerdPath) + requireParent(t, db, containerdPid, 1) + requireSessionLeader(t, db, containerdPid, containerdPid) + requireEntryLeader(t, db, containerdPid, containerdPid, Init) + + // containerd-shim-runc-v2 + requireProcess(t, db, containerdShimPid, containerdShimPath) + requireParent(t, db, containerdShimPid, 1) + requireSessionLeader(t, db, containerdShimPid, containerdPid) + requireEntryLeader(t, db, containerdShimPid, containerdPid, Init) + + // pause + requireProcess(t, db, pausePid, pausePath) + requireParent(t, db, pausePid, containerdShimPid) + requireSessionLeader(t, db, pausePid, pausePid) + requireEntryLeader(t, db, pausePid, pausePid, Container) + } +} + +// / (pid, sid, entry meta, entry leader) +// +// systemd (1, 1, none, none) +// +// \___ sshd (100, 100, "init", 100) +// \___ bash (1000, 1000, "sshd", 1000) +// \___ ls (1001, 1000, "sshd", 1000) +// | +// \___ grep (1002, 1000, "sshd", 1000) /* ppid/sid data is missing */ +// +// Grep does not have ppid or sid set, only pgid. Entry evaluation code +// should fallback to grabbing entry leader data from ls, the process group +// leader. +func testInitSshdBashLsAndGrepGrepOnlyHasGroupLeader(createDB createDBFn) testFn { + return func(t *testing.T) { + reader := procfs.NewMockReader() + populateProcfsWithInit(reader) + db := createDB(reader) + + sshdPid := uint32(100) + bashPid := uint32(1000) + lsPid := uint32(1001) + grepPid := uint32(1002) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: sshdPath, + Pids: types.PidInfo{ + Tgid: sshdPid, + Sid: sshdPid, + Ppid: 1, + }, + }) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: bashPath, + Pids: types.PidInfo{ + Tgid: bashPid, + Sid: bashPid, + Ppid: sshdPid, + }, + }) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: lsPath, + Pids: types.PidInfo{ + Tgid: lsPid, + Sid: bashPid, + Ppid: bashPid, + Pgid: lsPid, + }, + }) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: grepPath, + Pids: types.PidInfo{ + Tgid: grepPid, + Pgid: lsPid, + }, + }) + + // sshd + requireProcess(t, db, sshdPid, sshdPath) + requireParent(t, db, sshdPid, 1) + requireSessionLeader(t, db, sshdPid, sshdPid) + requireEntryLeader(t, db, sshdPid, sshdPid, Init) + + // bash + requireProcess(t, db, bashPid, bashPath) + requireParent(t, db, bashPid, sshdPid) + requireSessionLeader(t, db, bashPid, bashPid) + requireEntryLeader(t, db, bashPid, bashPid, Sshd) + + // ls + requireProcess(t, db, lsPid, lsPath) + requireParent(t, db, lsPid, bashPid) + requireSessionLeader(t, db, lsPid, bashPid) + requireEntryLeader(t, db, lsPid, bashPid, Sshd) + + // grep + grep, err := db.GetProcess(grepPid) + require.Nil(t, err) + requireParentUnset(t, grep) + + requireProcess(t, db, grepPid, grepPath) + requireEntryLeader(t, db, grepPid, bashPid, Sshd) + } +} + +// / (pid, sid, entry meta, entry leader) +// +// systemd (1, 1, none, none) +// +// \___ sshd (100, 100, "init", 100) +// \___ bash (1000, 1000, "sshd", 1000) +// \___ ls (1001, 1000, "sshd", 1000) +// | +// \___ grep (1002, 1000, "sshd", 1000) /* ppid/pgid data is missing */ +// +// Grep does not have ppid or pgid set, ppid. Entry evaluation code should +// fallback to grabbing entry leader data from sshd, the session leader. +func testInitSshdBashLsAndGrepGrepOnlyHasSessionLeader(createDB createDBFn) testFn { + return func(t *testing.T) { + reader := procfs.NewMockReader() + populateProcfsWithInit(reader) + db := createDB(reader) + + sshdPid := uint32(100) + bashPid := uint32(1000) + lsPid := uint32(1001) + grepPid := uint32(1002) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: sshdPath, + Pids: types.PidInfo{ + Tgid: sshdPid, + Sid: sshdPid, + Ppid: 1, + }, + }) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: bashPath, + Pids: types.PidInfo{ + Tgid: bashPid, + Sid: bashPid, + Ppid: sshdPid, + }, + }) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: lsPath, + Pids: types.PidInfo{ + Tgid: lsPid, + Sid: bashPid, + Ppid: bashPid, + Pgid: lsPid, + }, + }) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: grepPath, + Pids: types.PidInfo{ + Tgid: grepPid, + Sid: bashPid, + }, + }) + + // sshd + requireProcess(t, db, sshdPid, sshdPath) + requireParent(t, db, sshdPid, 1) + requireSessionLeader(t, db, sshdPid, sshdPid) + requireEntryLeader(t, db, sshdPid, sshdPid, Init) + + // bash + requireProcess(t, db, bashPid, bashPath) + requireParent(t, db, bashPid, sshdPid) + requireSessionLeader(t, db, bashPid, bashPid) + requireEntryLeader(t, db, bashPid, bashPid, Sshd) + + // ls + requireProcess(t, db, lsPid, lsPath) + requireParent(t, db, lsPid, bashPid) + requireSessionLeader(t, db, lsPid, bashPid) + requireEntryLeader(t, db, lsPid, bashPid, Sshd) + + // grep + grep, err := db.GetProcess(grepPid) + require.Nil(t, err) + requireParentUnset(t, grep) + + requireProcess(t, db, grepPid, grepPath) + requireSessionLeader(t, db, grepPid, bashPid) + requireEntryLeader(t, db, grepPid, bashPid, Sshd) + } +} + +// / (pid, sid, entry meta, entry leader) +// +// grep (1001, 1000, "unknown", 1001) +// +// No parent, session leader, or process group leader exists to draw +// on to get an entry leader for grep, fallback to assigning it an +// entry meta type of "unknown" and making it an entry leader. +func testGrepInIsolation(createDB createDBFn) testFn { + return func(t *testing.T) { + reader := procfs.NewMockReader() + db := createDB(reader) + + grepPid := uint32(1001) + + err := db.InsertExec(types.ProcessExecEvent{ + Filename: grepPath, + Pids: types.PidInfo{ + Tgid: grepPid, + Ppid: 1000, + Sid: grepPid, + }, + }) + require.Nil(t, err) + + process, err := db.GetProcess(grepPid) + require.Nil(t, err) + requireParentUnset(t, process) + + requireProcess(t, db, grepPid, grepPath) + requireSessionLeader(t, db, grepPid, grepPid) + requireEntryLeader(t, db, grepPid, grepPid, EntryUnknown) + } +} + +// / (pid, sid, entry meta, entry leader) +// +// kthreadd (2, 0, , ) +// +// \___ rcu_gp (3, 0, , ) +// +// Kernel threads should never have an entry meta type or entry leader set. +func testKernelThreads(createDB createDBFn) testFn { + return func(t *testing.T) { + reader := procfs.NewMockReader() + db := createDB(reader) + + kthreaddPid := uint32(2) + rcuGpPid := uint32(3) + + kthreaddPath := "kthreadd" + rcuGpPath := "rcu_gp" + + err := db.InsertExec(types.ProcessExecEvent{ + Filename: kthreaddPath, + Pids: types.PidInfo{ + Tgid: kthreaddPid, + Ppid: 1, + Sid: 0, + }, + }) + require.Nil(t, err) + + err = db.InsertExec(types.ProcessExecEvent{ + Filename: rcuGpPath, + Pids: types.PidInfo{ + Tgid: rcuGpPid, + Ppid: kthreaddPid, + Sid: 0, + }, + }) + require.Nil(t, err) + + // kthreadd + kthreadd, err := db.GetProcess(kthreaddPid) + require.Nil(t, err) + requireParentUnset(t, kthreadd) + requireSessionLeaderUnset(t, kthreadd) + requireEntryLeaderUnset(t, kthreadd) + + requireProcess(t, db, kthreaddPid, kthreaddPath) + + // rcu_gp + rcuGp, err := db.GetProcess(rcuGpPid) + require.Nil(t, err) + requireSessionLeaderUnset(t, rcuGp) + requireEntryLeaderUnset(t, rcuGp) + + requireProcess(t, db, rcuGpPid, rcuGpPath) + requireParent(t, db, rcuGpPid, kthreaddPid) + } +} From 6b7037b3fbecc4e4fab32b737c3e0d3f7a67ce30 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Mon, 15 Jan 2024 17:40:18 -0800 Subject: [PATCH 05/32] Update CHANGELOG --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 2114812d5365..1bc2cbd74c60 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -146,6 +146,7 @@ Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will d - Add `ignore_errors` option to audit module. {issue}15768[15768] {pull}36851[36851] - Fix copy arguments for strict aligned architectures. {pull}36976[36976] +- Added `add_session_metadata` processor, which enables session viewer on Auditbeat data. {pull}37640[37640] *Filebeat* From c688f5ad845cbac9cca7c9e7c9c7154231db5cf9 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Wed, 17 Jan 2024 13:56:17 -0800 Subject: [PATCH 06/32] Apply suggestions from code review Co-authored-by: Dan Kortschak <90160302+efd6@users.noreply.github.com> --- .../processors/add_session_metadata/add_session_metadata.go | 2 +- .../processors/add_session_metadata/pkg/processdb/db.go | 4 +++- .../processors/add_session_metadata/pkg/procfs/mock.go | 2 +- .../processors/add_session_metadata/pkg/procfs/procfs.go | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go b/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go index 0d88b55d4ca5..57d245d6f656 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go +++ b/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go @@ -130,7 +130,7 @@ func (p *addSessionMetadata) enrich(ev *beat.Event) (*beat.Event, error) { return result, nil } -//pidToUInt32 converts PID value to uint32 +// pidToUInt32 converts PID value to uint32 func pidToUInt32(value interface{}) (pid uint32, err error) { switch v := value.(type) { case string: diff --git a/x-pack/auditbeat/processors/add_session_metadata/pkg/processdb/db.go b/x-pack/auditbeat/processors/add_session_metadata/pkg/processdb/db.go index 30206e0b2ea5..4ed51ac3c00d 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/pkg/processdb/db.go +++ b/x-pack/auditbeat/processors/add_session_metadata/pkg/processdb/db.go @@ -78,7 +78,9 @@ func stringStartsWithEntryInList(str string, list []string) bool { } func isContainerRuntime(executable string) bool { - return stringStartsWithEntryInList(executable, containerRuntimes[:]) + return slices.ContainsFunc(containerRuntimes[:], func(s string) bool { + return strings.HasPrefix(executable, s) + }) } func isFilteredExecutable(executable string) bool { diff --git a/x-pack/auditbeat/processors/add_session_metadata/pkg/procfs/mock.go b/x-pack/auditbeat/processors/add_session_metadata/pkg/procfs/mock.go index cacb90a1fda4..bf3120add8d5 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/pkg/procfs/mock.go +++ b/x-pack/auditbeat/processors/add_session_metadata/pkg/procfs/mock.go @@ -31,7 +31,7 @@ func (r *MockReader) GetProcess(pid uint32) (ProcessInfo, error) { } func (r *MockReader) GetAllProcesses() ([]ProcessInfo, error) { - ret := make([]ProcessInfo, 0) + ret := make([]ProcessInfo, 0, len(r.entries)) for _, entry := range r.entries { ret = append(ret, entry) diff --git a/x-pack/auditbeat/processors/add_session_metadata/pkg/procfs/procfs.go b/x-pack/auditbeat/processors/add_session_metadata/pkg/procfs/procfs.go index 24832b7b9d35..388565e06912 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/pkg/procfs/procfs.go +++ b/x-pack/auditbeat/processors/add_session_metadata/pkg/procfs/procfs.go @@ -124,7 +124,7 @@ func (r ProcfsReader) getProcessInfo(proc p.Proc) (ProcessInfo, error) { // start time is needed to register the process in the database stat, err := proc.Stat() if err != nil { - return ProcessInfo{}, fmt.Errorf("failed to read /proc/%d/stat: %v", pid, err) + return ProcessInfo{}, fmt.Errorf("failed to read /proc/%d/stat: %w", pid, err) } argv, err := proc.CmdLine() From f35b4c7ce5fa89c0c7ef5a9378228c0007d9c8ad Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Wed, 17 Jan 2024 14:47:01 -0800 Subject: [PATCH 07/32] Rework directory structure --- .../add_session_metadata/add_session_metadata.go | 9 ++------- .../add_session_metadata_test.go | 4 ++-- .../{pkg => }/processdb/db.go | 2 ++ .../{pkg => }/processdb/db_test.go | 0 .../{pkg => }/processdb/entry_leader_test.go | 2 +- .../{pkg => }/processdb/simple.go | 4 ++-- .../{pkg => }/processdb/simple_test.go | 2 +- .../{pkg => }/procfs/mock.go | 0 .../{pkg => }/procfs/procfs.go | 16 ++++++++-------- .../provider/ebpf_provider/ebpf_provider.go | 2 +- .../{pkg => }/timeutils/time.go | 0 .../{pkg => }/timeutils/time_test.go | 0 12 files changed, 19 insertions(+), 22 deletions(-) rename x-pack/auditbeat/processors/add_session_metadata/{pkg => }/processdb/db.go (99%) rename x-pack/auditbeat/processors/add_session_metadata/{pkg => }/processdb/db_test.go (100%) rename x-pack/auditbeat/processors/add_session_metadata/{pkg => }/processdb/entry_leader_test.go (99%) rename x-pack/auditbeat/processors/add_session_metadata/{pkg => }/processdb/simple.go (99%) rename x-pack/auditbeat/processors/add_session_metadata/{pkg => }/processdb/simple_test.go (99%) rename x-pack/auditbeat/processors/add_session_metadata/{pkg => }/procfs/mock.go (100%) rename x-pack/auditbeat/processors/add_session_metadata/{pkg => }/procfs/procfs.go (94%) rename x-pack/auditbeat/processors/add_session_metadata/{pkg => }/timeutils/time.go (100%) rename x-pack/auditbeat/processors/add_session_metadata/{pkg => }/timeutils/time_test.go (100%) diff --git a/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go b/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go index 57d245d6f656..a9d333b3ce37 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go +++ b/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go @@ -13,10 +13,8 @@ import ( "github.com/elastic/elastic-agent-libs/mapstr" - "github.com/elastic/elastic-agent-libs/monitoring" - - "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/pkg/processdb" - "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/pkg/procfs" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/processdb" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/procfs" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/provider" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/provider/ebpf_provider" @@ -31,11 +29,8 @@ const ( logName = "processor." + processorName ) -var reg *monitoring.Registry - func init() { processors.RegisterPlugin(processorName, New) - reg = monitoring.Default.NewRegistry(logName, monitoring.DoNotReport) } type addSessionMetadata struct { diff --git a/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata_test.go b/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata_test.go index d476d558048f..e2089b8386ee 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata_test.go +++ b/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata_test.go @@ -9,8 +9,8 @@ import ( "testing" "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/pkg/processdb" - "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/pkg/procfs" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/processdb" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/procfs" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/types" "github.com/stretchr/testify/assert" diff --git a/x-pack/auditbeat/processors/add_session_metadata/pkg/processdb/db.go b/x-pack/auditbeat/processors/add_session_metadata/processdb/db.go similarity index 99% rename from x-pack/auditbeat/processors/add_session_metadata/pkg/processdb/db.go rename to x-pack/auditbeat/processors/add_session_metadata/processdb/db.go index 4ed51ac3c00d..8ece5038510f 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/pkg/processdb/db.go +++ b/x-pack/auditbeat/processors/add_session_metadata/processdb/db.go @@ -7,6 +7,8 @@ package processdb import ( "strings" + "slices" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/types" ) diff --git a/x-pack/auditbeat/processors/add_session_metadata/pkg/processdb/db_test.go b/x-pack/auditbeat/processors/add_session_metadata/processdb/db_test.go similarity index 100% rename from x-pack/auditbeat/processors/add_session_metadata/pkg/processdb/db_test.go rename to x-pack/auditbeat/processors/add_session_metadata/processdb/db_test.go diff --git a/x-pack/auditbeat/processors/add_session_metadata/pkg/processdb/entry_leader_test.go b/x-pack/auditbeat/processors/add_session_metadata/processdb/entry_leader_test.go similarity index 99% rename from x-pack/auditbeat/processors/add_session_metadata/pkg/processdb/entry_leader_test.go rename to x-pack/auditbeat/processors/add_session_metadata/processdb/entry_leader_test.go index ec2f4358f070..b8b5348df6a2 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/pkg/processdb/entry_leader_test.go +++ b/x-pack/auditbeat/processors/add_session_metadata/processdb/entry_leader_test.go @@ -10,7 +10,7 @@ import ( "github.com/stretchr/testify/require" - "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/pkg/procfs" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/procfs" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/types" // "github.com/elastic/elastic-agent-libs/logp" ) diff --git a/x-pack/auditbeat/processors/add_session_metadata/pkg/processdb/simple.go b/x-pack/auditbeat/processors/add_session_metadata/processdb/simple.go similarity index 99% rename from x-pack/auditbeat/processors/add_session_metadata/pkg/processdb/simple.go rename to x-pack/auditbeat/processors/add_session_metadata/processdb/simple.go index acf9de5bf7ff..353927a0695b 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/pkg/processdb/simple.go +++ b/x-pack/auditbeat/processors/add_session_metadata/processdb/simple.go @@ -17,8 +17,8 @@ import ( "sync" "time" - "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/pkg/procfs" - "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/pkg/timeutils" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/procfs" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/timeutils" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/types" "github.com/elastic/elastic-agent-libs/logp" ) diff --git a/x-pack/auditbeat/processors/add_session_metadata/pkg/processdb/simple_test.go b/x-pack/auditbeat/processors/add_session_metadata/processdb/simple_test.go similarity index 99% rename from x-pack/auditbeat/processors/add_session_metadata/pkg/processdb/simple_test.go rename to x-pack/auditbeat/processors/add_session_metadata/processdb/simple_test.go index a5df2cd869c6..41691e209462 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/pkg/processdb/simple_test.go +++ b/x-pack/auditbeat/processors/add_session_metadata/processdb/simple_test.go @@ -7,7 +7,7 @@ package processdb import ( "testing" - "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/pkg/procfs" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/procfs" "github.com/stretchr/testify/assert" "github.com/elastic/elastic-agent-libs/logp" diff --git a/x-pack/auditbeat/processors/add_session_metadata/pkg/procfs/mock.go b/x-pack/auditbeat/processors/add_session_metadata/procfs/mock.go similarity index 100% rename from x-pack/auditbeat/processors/add_session_metadata/pkg/procfs/mock.go rename to x-pack/auditbeat/processors/add_session_metadata/procfs/mock.go diff --git a/x-pack/auditbeat/processors/add_session_metadata/pkg/procfs/procfs.go b/x-pack/auditbeat/processors/add_session_metadata/procfs/procfs.go similarity index 94% rename from x-pack/auditbeat/processors/add_session_metadata/pkg/procfs/procfs.go rename to x-pack/auditbeat/processors/add_session_metadata/procfs/procfs.go index 388565e06912..2496479abe1c 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/pkg/procfs/procfs.go +++ b/x-pack/auditbeat/processors/add_session_metadata/procfs/procfs.go @@ -9,9 +9,9 @@ import ( "strconv" "strings" - p "github.com/prometheus/procfs" + "github.com/prometheus/procfs" - "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/pkg/timeutils" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/timeutils" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/types" "github.com/elastic/elastic-agent-libs/logp" @@ -42,7 +42,7 @@ func NewProcfsReader(logger logp.Logger) ProcfsReader { } } -type Stat p.ProcStat +type Stat procfs.ProcStat type ProcessInfo struct { Pids types.PidInfo @@ -55,7 +55,7 @@ type ProcessInfo struct { CGroupPath string } -func credsFromProc(proc p.Proc) (types.CredInfo, error) { +func credsFromProc(proc procfs.Proc) (types.CredInfo, error) { status, err := proc.NewStatus() if err != nil { return types.CredInfo{}, err @@ -118,7 +118,7 @@ func credsFromProc(proc p.Proc) (types.CredInfo, error) { }, nil } -func (r ProcfsReader) getProcessInfo(proc p.Proc) (ProcessInfo, error) { +func (r ProcfsReader) getProcessInfo(proc procfs.Proc) (ProcessInfo, error) { pid := uint32(proc.PID) // All other info can be best effort, but failing to get pid info and // start time is needed to register the process in the database @@ -200,7 +200,7 @@ func (r ProcfsReader) getProcessInfo(proc p.Proc) (ProcessInfo, error) { } func (r ProcfsReader) GetProcess(pid uint32) (ProcessInfo, error) { - proc, err := p.NewProc(int(pid)) + proc, err := procfs.NewProc(int(pid)) if err != nil { return ProcessInfo{}, err } @@ -208,7 +208,7 @@ func (r ProcfsReader) GetProcess(pid uint32) (ProcessInfo, error) { } // returns empty slice on error func (r ProcfsReader) GetAllProcesses() ([]ProcessInfo, error) { - procs, err := p.AllProcs() + procs, err := procfs.AllProcs() if err != nil { return nil, err } @@ -226,7 +226,7 @@ func (r ProcfsReader) GetAllProcesses() ([]ProcessInfo, error) { } func (r ProcfsReader) getEnviron(pid uint32) (map[string]string, error) { - proc, err := p.NewProc(int(pid)) + proc, err := procfs.NewProc(int(pid)) if err != nil { return nil, err } diff --git a/x-pack/auditbeat/processors/add_session_metadata/provider/ebpf_provider/ebpf_provider.go b/x-pack/auditbeat/processors/add_session_metadata/provider/ebpf_provider/ebpf_provider.go index c83fcf37cda3..4bcf73d4b679 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/provider/ebpf_provider/ebpf_provider.go +++ b/x-pack/auditbeat/processors/add_session_metadata/provider/ebpf_provider/ebpf_provider.go @@ -11,7 +11,7 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/pkg/processdb" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/processdb" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/provider" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/types" diff --git a/x-pack/auditbeat/processors/add_session_metadata/pkg/timeutils/time.go b/x-pack/auditbeat/processors/add_session_metadata/timeutils/time.go similarity index 100% rename from x-pack/auditbeat/processors/add_session_metadata/pkg/timeutils/time.go rename to x-pack/auditbeat/processors/add_session_metadata/timeutils/time.go diff --git a/x-pack/auditbeat/processors/add_session_metadata/pkg/timeutils/time_test.go b/x-pack/auditbeat/processors/add_session_metadata/timeutils/time_test.go similarity index 100% rename from x-pack/auditbeat/processors/add_session_metadata/pkg/timeutils/time_test.go rename to x-pack/auditbeat/processors/add_session_metadata/timeutils/time_test.go From bf38e893340b59c04e4b63bb3620e32423f291f0 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Wed, 17 Jan 2024 14:56:31 -0800 Subject: [PATCH 08/32] Remove DB interface Remove the DB interface, as there will only be one implementation for it --- .../add_session_metadata.go | 6 +- .../add_session_metadata_test.go | 2 +- .../add_session_metadata/processdb/db.go | 657 +++++++++++++++++- .../add_session_metadata/processdb/db_test.go | 121 ++++ .../add_session_metadata/processdb/simple.go | 657 ------------------ .../processdb/simple_test.go | 131 ---- .../provider/ebpf_provider/ebpf_provider.go | 8 +- 7 files changed, 774 insertions(+), 808 deletions(-) delete mode 100644 x-pack/auditbeat/processors/add_session_metadata/processdb/simple.go delete mode 100644 x-pack/auditbeat/processors/add_session_metadata/processdb/simple_test.go diff --git a/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go b/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go index a9d333b3ce37..31aa881b6929 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go +++ b/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go @@ -36,7 +36,7 @@ func init() { type addSessionMetadata struct { config Config logger *logp.Logger - db processdb.DB + db *processdb.DB provider provider.Provider } @@ -50,14 +50,14 @@ func New(cfg *config.C) (beat.Processor, error) { ctx := context.TODO() reader := procfs.NewProcfsReader(*logger) - db := processdb.NewSimpleDB(reader, *logger) + db := processdb.NewDB(reader, *logger) backfilledPIDs := db.ScrapeProcfs() logger.Debugf("backfilled %d processes", len(backfilledPIDs)) switch c.Backend { case "ebpf": - p, err := ebpf_provider.NewProvider(ctx, *logger, db) + p, err := ebpf_provider.NewProvider(ctx, logger, db) if err != nil { return nil, fmt.Errorf("failed to create ebpf provider: %w", err) } diff --git a/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata_test.go b/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata_test.go index e2089b8386ee..188e0c1d23b7 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata_test.go +++ b/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata_test.go @@ -156,7 +156,7 @@ var ( func TestEnrich(t *testing.T) { for _, tt := range enrichTests { reader := procfs.NewMockReader() - db := processdb.NewSimpleDB(reader, *logger) + db := processdb.NewDB(reader, *logger) for _, ev := range tt.mockProcesses { err := db.InsertExec(ev) diff --git a/x-pack/auditbeat/processors/add_session_metadata/processdb/db.go b/x-pack/auditbeat/processors/add_session_metadata/processdb/db.go index 8ece5038510f..12992cd94bd7 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/processdb/db.go +++ b/x-pack/auditbeat/processors/add_session_metadata/processdb/db.go @@ -5,23 +5,25 @@ package processdb import ( - "strings" - + "encoding/base64" + "errors" + "fmt" + "math/bits" + "os" + "path" "slices" + "sort" + "strconv" + "strings" + "sync" + "time" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/procfs" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/timeutils" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/types" + "github.com/elastic/elastic-agent-libs/logp" ) -type DB interface { - InsertFork(fork types.ProcessForkEvent) error - InsertExec(exec types.ProcessExecEvent) error - InsertSetsid(setsid types.ProcessSetsidEvent) error - InsertExit(exit types.ProcessExitEvent) error - GetProcess(pid uint32) (types.Process, error) - GetEntryType(pid uint32) (EntryType, error) - ScrapeProcfs() []uint32 -} - type TtyType int const ( @@ -69,6 +71,637 @@ const ( ttyMaxMinor = 255 ) +type Process struct { + Pids types.PidInfo + Creds types.CredInfo + CTty types.TtyDev + Argv []string + Cwd string + Env map[string]string + Filename string +} + +var ( + // The contents of these two files are needed to calculate entity IDs. + // Fail fast on startup if we can't read them. + bootID = mustReadBootID() + pidNsInode = mustReadPidNsInode() + capNames = []string{ + "CAP_CHOWN", // 0 + "CAP_DAC_OVERRIDE", // 1 + "CAP_DAC_READ_SEARCH", // 2 + "CAP_FOWNER", // 3 + "CAP_FSETID", // 4 + "CAP_KILL", // 5 + "CAP_SETGID", // 6 + "CAP_SETUID", // 7 + "CAP_SETPCAP", // 8 + "CAP_LINUX_IMMUTABLE", // 9 + "CAP_NET_BIND_SERVICE", // 10 + "CAP_NET_BROADCAST", // 11 + "CAP_NET_ADMIN", // 12 + "CAP_NET_RAW", // 13 + "CAP_IPC_LOCK", // 14 + "CAP_IPC_OWNER", // 15 + "CAP_SYS_MODULE", // 16 + "CAP_SYS_RAWIO", // 17 + "CAP_SYS_CHROOT", // 18 + "CAP_SYS_PTRACE", // 19 + "CAP_SYS_PACCT", // 20 + "CAP_SYS_ADMIN", // 21 + "CAP_SYS_BOOT", // 22 + "CAP_SYS_NICE", // 23 + "CAP_SYS_RESOURCE", // 24 + "CAP_SYS_TIME", // 25 + "CAP_SYS_TTY_CONFIG", // 26 + "CAP_MKNOD", // 27 + "CAP_LEASE", // 28 + "CAP_AUDIT_WRITE", // 29 + "CAP_AUDIT_CONTROL", // 30 + "CAP_SETFCAP", // 31 + "CAP_MAC_OVERRIDE", // 32 + "CAP_MAC_ADMIN", // 33 + "CAP_SYSLOG", // 34 + "CAP_WAKE_ALARM", // 35 + "CAP_BLOCK_SUSPEND", // 36 + "CAP_AUDIT_READ", // 37 + "CAP_PERFMON", // 38 + "CAP_BPF", // 39 + "CAP_CHECKPOINT_RESTORE", // 40 + // The ECS spec allows for numerical string representation. + // The following capability values are not assigned as of Dec 28, 2023. + // If they are added in a future kernel, and this slice has not been + // updated, the numerical string will used. + "41", + "42", + "43", + "44", + "45", + "46", + "47", + "48", + "49", + "50", + "51", + "52", + "53", + "54", + "55", + "56", + "57", + "58", + "59", + "60", + "61", + "62", + "63", + } +) + +func mustReadBootID() string { + bootID, err := os.ReadFile("/proc/sys/kernel/random/boot_id") + if err != nil { + panic(fmt.Sprintf("could not read /proc/sys/kernel/random/boot_id: %v", err)) + } + + return strings.TrimRight(string(bootID), "\n") +} + +func mustReadPidNsInode() uint64 { + var ret uint64 + + pidNsInodeRaw, err := os.Readlink("/proc/self/ns/pid") + if err != nil { + panic(fmt.Sprintf("could not read /proc/self/ns/pid: %v", err)) + } + + if _, err = fmt.Sscanf(pidNsInodeRaw, "pid:[%d]", &ret); err != nil { + panic(fmt.Sprintf("could not parse contents of /proc/self/ns/pid (%s): %v", pidNsInodeRaw, err)) + } + + return ret +} + +func pidInfoFromProto(p types.PidInfo) types.PidInfo { + return types.PidInfo{ + StartTimeNs: p.StartTimeNs, + Tid: p.Tid, + Tgid: p.Tgid, + Vpid: p.Vpid, + Ppid: p.Ppid, + Pgid: p.Pgid, + Sid: p.Sid, + } +} + +func credInfoFromProto(p types.CredInfo) types.CredInfo { + return types.CredInfo{ + Ruid: p.Ruid, + Rgid: p.Rgid, + Euid: p.Euid, + Egid: p.Egid, + Suid: p.Suid, + Sgid: p.Sgid, + CapPermitted: p.CapPermitted, + CapEffective: p.CapEffective, + } +} + +func ttyTermiosFromProto(p types.TtyTermios) types.TtyTermios { + return types.TtyTermios{ + CIflag: p.CIflag, + COflag: p.COflag, + CLflag: p.CLflag, + CCflag: p.CCflag, + } +} + +func ttyWinsizeFromProto(p types.TtyWinsize) types.TtyWinsize { + return types.TtyWinsize{ + Rows: p.Rows, + Cols: p.Cols, + } +} + +func ttyDevFromProto(p types.TtyDev) types.TtyDev { + return types.TtyDev{ + Major: p.Major, + Minor: p.Minor, + Winsize: ttyWinsizeFromProto(p.Winsize), + Termios: ttyTermiosFromProto(p.Termios), + } +} + +type DB struct { + sync.RWMutex + logger *logp.Logger + processes map[uint32]Process + entryLeaders map[uint32]EntryType + entryLeaderRelationships map[uint32]uint32 + procfs procfs.Reader +} + +func NewDB(reader procfs.Reader, logger logp.Logger) *DB { + return &DB{ + logger: logp.NewLogger("processdb"), + processes: make(map[uint32]Process), + entryLeaders: make(map[uint32]EntryType), + entryLeaderRelationships: make(map[uint32]uint32), + procfs: reader, + } +} + +func (db *DB) calculateEntityIDv1(pid uint32, startTime time.Time) string { + return base64.StdEncoding.EncodeToString( + []byte( + fmt.Sprintf("%d__%s__%d__%d", + pidNsInode, + bootID, + uint64(pid), + uint64(startTime.Unix()), + ), + ), + ) +} + +// `path.Base` returns a '.' for empty strings, this just special cases that +// situation to return an empty string +func basename(pathStr string) string { + if pathStr == "" { + return "" + } + + return path.Base(pathStr) +} + +func (db *DB) InsertFork(fork types.ProcessForkEvent) error { + db.Lock() + defer db.Unlock() + + pid := fork.ChildPids.Tgid + ppid := fork.ParentPids.Tgid + if entry, ok := db.processes[ppid]; ok { + entry.Pids = pidInfoFromProto(fork.ChildPids) + entry.Creds = credInfoFromProto(fork.Creds) + db.processes[pid] = entry + if entryPid, ok := db.entryLeaderRelationships[ppid]; ok { + db.entryLeaderRelationships[pid] = entryPid + } + } else { + db.processes[pid] = Process{ + Pids: pidInfoFromProto(fork.ChildPids), + Creds: credInfoFromProto(fork.Creds), + } + } + + return nil +} + +func (db *DB) insertProcess(process Process) { + pid := process.Pids.Tgid + db.processes[pid] = process + entryLeaderPid := db.evaluateEntryLeader(process) + if entryLeaderPid != nil { + db.entryLeaderRelationships[pid] = *entryLeaderPid + db.logger.Debugf("%v name: %s, entry_leader: %d, entry_type: %s", process.Pids, process.Filename, *entryLeaderPid, string(db.entryLeaders[*entryLeaderPid])) + } else { + db.logger.Debugf("%v name: %s, NO ENTRY LEADER", process.Pids, process.Filename) + } +} + +func (db *DB) InsertExec(exec types.ProcessExecEvent) error { + db.Lock() + defer db.Unlock() + + proc := Process{ + Pids: pidInfoFromProto(exec.Pids), + Creds: credInfoFromProto(exec.Creds), + CTty: ttyDevFromProto(exec.CTty), + Argv: exec.Argv, + Cwd: exec.Cwd, + Env: exec.Env, + Filename: exec.Filename, + } + + db.processes[exec.Pids.Tgid] = proc + entryLeaderPid := db.evaluateEntryLeader(proc) + if entryLeaderPid != nil { + db.entryLeaderRelationships[exec.Pids.Tgid] = *entryLeaderPid + } + + return nil +} + +func (db *DB) createEntryLeader(pid uint32, entryType EntryType) { + db.entryLeaders[pid] = entryType + db.logger.Debugf("created entry leader %d: %s, name: %s", pid, string(entryType), db.processes[pid].Filename) +} + +// pid returned is a pointer type because its possible for no +func (db *DB) evaluateEntryLeader(p Process) *uint32 { + pid := p.Pids.Tgid + + // init never has an entry leader or meta type + if p.Pids.Tgid == 1 { + db.logger.Debugf("entry_eval %d: process is init, no entry type", p.Pids.Tgid) + return nil + } + + // kernel threads also never have an entry leader or meta type kthreadd + // (always pid 2) is the parent of all kernel threads, by filtering pid == + // 2 || ppid == 2, we get rid of all of them + if p.Pids.Tgid == 2 || p.Pids.Ppid == 2 { + db.logger.Debugf("entry_eval %d: kernel threads never an entry type (parent is pid 2)", p.Pids.Tgid) + return nil + } + + // could be an entry leader + if p.Pids.Tgid == p.Pids.Sid { + ttyType := getTtyType(p.CTty.Major, p.CTty.Minor) + + procBasename := basename(p.Filename) + if ttyType == Tty { + db.createEntryLeader(pid, Terminal) + db.logger.Debugf("entry_eval %d: entry type is terminal", p.Pids.Tgid) + return &pid + } else if ttyType == TtyConsole && procBasename == "login" { + db.createEntryLeader(pid, EntryConsole) + db.logger.Debugf("entry_eval %d: entry type is console", p.Pids.Tgid) + return &pid + } else if p.Pids.Ppid == 1 { + db.createEntryLeader(pid, Init) + db.logger.Debugf("entry_eval %d: entry type is init", p.Pids.Tgid) + return &pid + } else if !isFilteredExecutable(procBasename) { + if parent, ok := db.processes[p.Pids.Ppid]; ok { + parentBasename := basename(parent.Filename) + if ttyType == Pts && parentBasename == "ssm-session-worker" { + db.createEntryLeader(pid, Ssm) + db.logger.Debugf("entry_eval %d: entry type is ssm", p.Pids.Tgid) + return &pid + } else if parentBasename == "sshd" && procBasename != "sshd" { + // TODO: get ip from env vars + db.createEntryLeader(pid, Sshd) + db.logger.Debugf("entry_eval %d: entry type is sshd", p.Pids.Tgid) + return &pid + } else if isContainerRuntime(parentBasename) { + db.createEntryLeader(pid, Container) + db.logger.Debugf("entry_eval %d: entry type is container", p.Pids.Tgid) + return &pid + } + } + } else { + db.logger.Debugf("entry_eval %d: is a filtered executable: %s", p.Pids.Tgid, procBasename) + } + } + + // if not a session leader or was not determined to be an entry leader, get + // it via parent, session leader, group leader (in that order) + relations := []struct { + pid uint32 + name string + }{ + { + pid: p.Pids.Ppid, + name: "parent", + }, + { + pid: p.Pids.Sid, + name: "session_leader", + }, + { + pid: p.Pids.Pgid, + name: "group_leader", + }, + } + + for _, relation := range relations { + if entry, ok := db.entryLeaderRelationships[relation.pid]; ok { + entryType := db.entryLeaders[entry] + db.logger.Debugf("entry_eval %d: got entry_leader: %d (%s), from relative: %d (%s)", p.Pids.Tgid, entry, string(entryType), relation.pid, relation.name) + return &entry + } else { + db.logger.Debugf("entry_eval %d: failed to find relative: %d (%s)", p.Pids.Tgid, relation.pid, relation.name) + } + } + + // if it's a session leader, then make it its own entry leader with unknown + // entry type + if p.Pids.Tgid == p.Pids.Sid { + db.createEntryLeader(pid, EntryUnknown) + db.logger.Debugf("entry_eval %d: this is a session leader and no relative has an entry leader. entry type is unknown", p.Pids.Tgid) + return &pid + } + + db.logger.Debugf("entry_eval %d: this is not a session leader and no relative has an entry leader, entry_leader will be unset", p.Pids.Tgid) + return nil +} + +func (db *DB) InsertSetsid(setsid types.ProcessSetsidEvent) error { + db.Lock() + defer db.Unlock() + + if entry, ok := db.processes[setsid.Pids.Tgid]; ok { + entry.Pids = pidInfoFromProto(setsid.Pids) + db.processes[setsid.Pids.Tgid] = entry + } else { + db.processes[setsid.Pids.Tgid] = Process{ + Pids: pidInfoFromProto(setsid.Pids), + } + } + + return nil +} + +func (db *DB) InsertExit(exit types.ProcessExitEvent) error { + db.Lock() + defer db.Unlock() + + pid := exit.Pids.Tgid + delete(db.processes, pid) + delete(db.entryLeaders, pid) + delete(db.entryLeaderRelationships, pid) + return nil +} + +// TODO: is this the correct definition? I looked in endpoint and I swear it looks too simple/generalized +func interactiveFromTty(tty types.TtyDev) bool { + return TtyUnknown != getTtyType(tty.Major, tty.Minor) +} + +func ecsCapsFromU64(capabilities uint64) []string { + var ecsCaps []string + if c := bits.OnesCount64(capabilities); c > 0 { + ecsCaps = make([]string, 0, c) + } + for bitnum := 0; bitnum < 64; bitnum++ { + if (capabilities & (1 << bitnum)) > 0 { + ecsCaps = append(ecsCaps, capNames[bitnum]) + } + } + return ecsCaps +} + +func fullProcessFromDBProcess(p Process) types.Process { + reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(p.Pids.StartTimeNs) + interactive := interactiveFromTty(p.CTty) + + ret := types.Process{ + PID: p.Pids.Tgid, + Start: timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime), + Name: basename(p.Filename), + Executable: p.Filename, + Args: p.Argv, + WorkingDirectory: p.Cwd, + Interactive: &interactive, + } + + euid := p.Creds.Euid + egid := p.Creds.Egid + ret.User.ID = strconv.FormatUint(uint64(euid), 10) + ret.Group.ID = strconv.FormatUint(uint64(egid), 10) + ret.Thread.Capabilities.Permitted = ecsCapsFromU64(p.Creds.CapPermitted) + ret.Thread.Capabilities.Effective = ecsCapsFromU64(p.Creds.CapEffective) + + return ret +} + +func fillParent(process *types.Process, parent Process) { + reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(parent.Pids.StartTimeNs) + + interactive := interactiveFromTty(parent.CTty) + euid := parent.Creds.Euid + egid := parent.Creds.Egid + process.Parent.PID = parent.Pids.Tgid + process.Parent.Start = timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime) + process.Parent.Name = basename(parent.Filename) + process.Parent.Executable = parent.Filename + process.Parent.Args = parent.Argv + process.Parent.WorkingDirectory = parent.Cwd + process.Parent.Interactive = &interactive + process.Parent.User.ID = strconv.FormatUint(uint64(euid), 10) + process.Parent.Group.ID = strconv.FormatUint(uint64(egid), 10) +} + +func fillGroupLeader(process *types.Process, groupLeader Process) { + reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(groupLeader.Pids.StartTimeNs) + + interactive := interactiveFromTty(groupLeader.CTty) + euid := groupLeader.Creds.Euid + egid := groupLeader.Creds.Egid + process.GroupLeader.PID = groupLeader.Pids.Tgid + process.GroupLeader.Start = timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime) + process.GroupLeader.Name = basename(groupLeader.Filename) + process.GroupLeader.Executable = groupLeader.Filename + process.GroupLeader.Args = groupLeader.Argv + process.GroupLeader.WorkingDirectory = groupLeader.Cwd + process.GroupLeader.Interactive = &interactive + process.GroupLeader.User.ID = strconv.FormatUint(uint64(euid), 10) + process.GroupLeader.Group.ID = strconv.FormatUint(uint64(egid), 10) +} + +func fillSessionLeader(process *types.Process, sessionLeader Process) { + reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(sessionLeader.Pids.StartTimeNs) + + interactive := interactiveFromTty(sessionLeader.CTty) + euid := sessionLeader.Creds.Euid + egid := sessionLeader.Creds.Egid + process.SessionLeader.PID = sessionLeader.Pids.Tgid + process.SessionLeader.Start = timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime) + process.SessionLeader.Name = basename(sessionLeader.Filename) + process.SessionLeader.Executable = sessionLeader.Filename + process.SessionLeader.Args = sessionLeader.Argv + process.SessionLeader.WorkingDirectory = sessionLeader.Cwd + process.SessionLeader.Interactive = &interactive + process.SessionLeader.User.ID = strconv.FormatUint(uint64(euid), 10) + process.SessionLeader.Group.ID = strconv.FormatUint(uint64(egid), 10) +} + +func fillEntryLeader(process *types.Process, entryType EntryType, entryLeader Process) { + reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(entryLeader.Pids.StartTimeNs) + + interactive := interactiveFromTty(entryLeader.CTty) + euid := entryLeader.Creds.Euid + egid := entryLeader.Creds.Egid + process.EntryLeader.PID = entryLeader.Pids.Tgid + process.EntryLeader.Start = timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime) + process.EntryLeader.Name = basename(entryLeader.Filename) + process.EntryLeader.Executable = entryLeader.Filename + process.EntryLeader.Args = entryLeader.Argv + process.EntryLeader.WorkingDirectory = entryLeader.Cwd + process.EntryLeader.Interactive = &interactive + process.EntryLeader.User.ID = strconv.FormatUint(uint64(euid), 10) + process.EntryLeader.Group.ID = strconv.FormatUint(uint64(egid), 10) + + process.EntryLeader.EntryMeta.Type = string(entryType) +} + +func (db *DB) setEntityID(process *types.Process) { + if process.PID != 0 && process.Start != nil { + process.EntityID = db.calculateEntityIDv1(process.PID, *process.Start) + } + + if process.Parent.PID != 0 && process.Parent.Start != nil { + process.Parent.EntityID = db.calculateEntityIDv1(process.Parent.PID, *process.Parent.Start) + } + + if process.GroupLeader.PID != 0 && process.GroupLeader.Start != nil { + process.GroupLeader.EntityID = db.calculateEntityIDv1(process.GroupLeader.PID, *process.GroupLeader.Start) + } + + if process.SessionLeader.PID != 0 && process.SessionLeader.Start != nil { + process.SessionLeader.EntityID = db.calculateEntityIDv1(process.SessionLeader.PID, *process.SessionLeader.Start) + } + + if process.EntryLeader.PID != 0 && process.EntryLeader.Start != nil { + process.EntryLeader.EntityID = db.calculateEntityIDv1(process.EntryLeader.PID, *process.EntryLeader.Start) + } +} + +func setSameAsProcess(process *types.Process) { + if process.GroupLeader.PID != 0 && process.GroupLeader.Start != nil { + sameAsProcess := process.PID == process.GroupLeader.PID + process.GroupLeader.SameAsProcess = &sameAsProcess + } + + if process.SessionLeader.PID != 0 && process.SessionLeader.Start != nil { + sameAsProcess := process.PID == process.SessionLeader.PID + process.SessionLeader.SameAsProcess = &sameAsProcess + } + + if process.EntryLeader.PID != 0 && process.EntryLeader.Start != nil { + sameAsProcess := process.PID == process.EntryLeader.PID + process.EntryLeader.SameAsProcess = &sameAsProcess + } +} + +func (db *DB) GetProcess(pid uint32) (types.Process, error) { + db.RLock() + defer db.RUnlock() + + process, ok := db.processes[pid] + if !ok { + return types.Process{}, errors.New("process not found") + } + + ret := fullProcessFromDBProcess(process) + + if parent, ok := db.processes[process.Pids.Ppid]; ok { + fillParent(&ret, parent) + } + + if groupLeader, ok := db.processes[process.Pids.Pgid]; ok { + fillGroupLeader(&ret, groupLeader) + } + + if sessionLeader, ok := db.processes[process.Pids.Sid]; ok { + fillSessionLeader(&ret, sessionLeader) + } + + if entryLeaderPid, foundEntryLeaderPid := db.entryLeaderRelationships[process.Pids.Tgid]; foundEntryLeaderPid { + if entryLeader, foundEntryLeader := db.processes[entryLeaderPid]; foundEntryLeader { + // if there is an entry leader then there is a matching member in the entryLeaders table + fillEntryLeader(&ret, db.entryLeaders[entryLeaderPid], entryLeader) + } else { + db.logger.Errorf("failed to find entry leader entry %d for %d (%s)", entryLeaderPid, pid, db.processes[pid].Filename) + } + } else { + db.logger.Errorf("failed to find entry leader for %d (%s)", pid, db.processes[pid].Filename) + } + + db.setEntityID(&ret) + setSameAsProcess(&ret) + + return ret, nil +} + +func (db *DB) GetEntryType(pid uint32) (EntryType, error) { + db.RLock() + defer db.RUnlock() + + if entryType, ok := db.entryLeaders[pid]; ok { + return entryType, nil + } + return EntryUnknown, nil +} + +func (db *DB) ScrapeProcfs() []uint32 { + db.Lock() + defer db.Unlock() + + procs, err := db.procfs.GetAllProcesses() + if err != nil { + db.logger.Errorf("failed to get processes from procfs: %v", err) + return make([]uint32, 0) + } + + // sorting the slice to make sure that parents, session leaders, group + // leaders come first in the queue + sort.Slice(procs, func(i, j int) bool { + return procs[i].Pids.Tgid == procs[j].Pids.Ppid || + procs[i].Pids.Tgid == procs[j].Pids.Sid || + procs[i].Pids.Tgid == procs[j].Pids.Pgid + }) + + pids := make([]uint32, 0) + for _, procInfo := range procs { + process := Process{ + Pids: pidInfoFromProto(procInfo.Pids), + Creds: credInfoFromProto(procInfo.Creds), + CTty: ttyDevFromProto(procInfo.CTty), + Argv: procInfo.Argv, + Cwd: procInfo.Cwd, + Env: procInfo.Env, + Filename: procInfo.Filename, + } + + db.insertProcess(process) + pids = append(pids, process.Pids.Tgid) + } + + return pids +} + func stringStartsWithEntryInList(str string, list []string) bool { for _, entry := range list { if strings.HasPrefix(str, entry) { diff --git a/x-pack/auditbeat/processors/add_session_metadata/processdb/db_test.go b/x-pack/auditbeat/processors/add_session_metadata/processdb/db_test.go index 1d2428430e2a..2450eb7ee7be 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/processdb/db_test.go +++ b/x-pack/auditbeat/processors/add_session_metadata/processdb/db_test.go @@ -7,12 +7,133 @@ package processdb import ( "testing" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/procfs" "github.com/stretchr/testify/assert" + + "github.com/elastic/elastic-agent-libs/logp" + + "golang.org/x/sys/unix" ) +var logger = logp.NewLogger("processdb") +var reader = procfs.NewMockReader() + +// glue function to fit the return type required by these tests +func newDBIntf(reader procfs.Reader) DB { + ret := NewDB(reader, *logger) + _ = ret.ScrapeProcfs() + return *ret +} + + func TestGetTtyType(t *testing.T) { assert.Equal(t, TtyConsole, getTtyType(4, 0)) assert.Equal(t, Pts, getTtyType(136, 0)) assert.Equal(t, Tty, getTtyType(4, 64)) assert.Equal(t, TtyUnknown, getTtyType(1000, 1000)) } + +func TestSingleProcessSessionLeaderEntryTypeTerminal(t *testing.T) { + testSingleProcessSessionLeaderEntryTypeTerminal(newDBIntf)(t) +} + +func TestSingleProcessSessionLeaderLoginProcess(t *testing.T) { + testSingleProcessSessionLeaderLoginProcess(newDBIntf)(t) +} + +func TestSingleProcessSessionLeaderChildOfInit(t *testing.T) { + testSingleProcessSessionLeaderChildOfInit(newDBIntf)(t) +} + +func TestSingleProcessSessionLeaderChildOfSsmSessionWorker(t *testing.T) { + testSingleProcessSessionLeaderChildOfSsmSessionWorker(newDBIntf)(t) +} + +func TestSingleProcessSessionLeaderChildOfSshd(t *testing.T) { + testSingleProcessSessionLeaderChildOfSshd(newDBIntf)(t) +} + +func TestSingleProcessSessionLeaderChildOfContainerdShim(t *testing.T) { + testSingleProcessSessionLeaderChildOfContainerdShim(newDBIntf)(t) +} + +func TestSingleProcessSessionLeaderOfRunc(t *testing.T) { + testSingleProcessSessionLeaderChildOfRunc(newDBIntf)(t) +} + +func TestSingleProcessEmptyProcess(t *testing.T) { + testSingleProcessEmptyProcess(newDBIntf)(t) +} + +func TestSingleProcessOverwriteOldEntryLeader(t *testing.T) { + testSingleProcessOverwriteOldEntryLeader(newDBIntf)(t) +} + +func TestInitSshdBashLs(t *testing.T) { + testInitSshdBashLs(newDBIntf)(t) +} + +func TestInitSshdSshdBashLs(t *testing.T) { + testInitSshdSshdBashLs(newDBIntf)(t) +} + +func TestInitSshdSshdSshdBashLs(t *testing.T) { + testInitSshdSshdSshdBashLs(newDBIntf)(t) +} + +func TestInitContainerdContainerdShim(t *testing.T) { + testInitContainerdContainerdShim(newDBIntf)(t) +} + +func TestInitContainerdShimBashContainerdShimIsReparentedToInit(t *testing.T) { + testInitContainerdShimBashContainerdShimIsReparentedToInit(newDBIntf)(t) +} + +func TestInitContainerdShimPauseContainerdShimIsReparentedToInit(t *testing.T) { + testInitContainerdShimPauseContainerdShimIsReparentedToInit(newDBIntf)(t) +} + +func TestInitSshdBashLsAndGrepGrepOnlyHasGroupLeader(t *testing.T) { + testInitSshdBashLsAndGrepGrepOnlyHasGroupLeader(newDBIntf)(t) +} + +func TestInitSshdBashLsAndGrepGrepOnlyHasSessionLeader(t *testing.T) { + testInitSshdBashLsAndGrepGrepOnlyHasSessionLeader(newDBIntf)(t) +} + +func TestGrepInIsolation(t *testing.T) { + testGrepInIsolation(newDBIntf)(t) +} + +func TestKernelThreads(t *testing.T) { + testKernelThreads(newDBIntf)(t) +} + +func TestCapsFromU64ToECS(t *testing.T) { + expected := []string{"CAP_CHOWN"} + assert.Equal(t, expected, ecsCapsFromU64(uint64(1< 0 { - ecsCaps = make([]string, 0, c) - } - for bitnum := 0; bitnum < 64; bitnum++ { - if (capabilities & (1 << bitnum)) > 0 { - ecsCaps = append(ecsCaps, capNames[bitnum]) - } - } - return ecsCaps -} - -func fullProcessFromDBProcess(p Process) types.Process { - reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(p.Pids.StartTimeNs) - interactive := interactiveFromTty(p.CTty) - - ret := types.Process{ - PID: p.Pids.Tgid, - Start: timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime), - Name: basename(p.Filename), - Executable: p.Filename, - Args: p.Argv, - WorkingDirectory: p.Cwd, - Interactive: &interactive, - } - - euid := p.Creds.Euid - egid := p.Creds.Egid - ret.User.ID = strconv.FormatUint(uint64(euid), 10) - ret.Group.ID = strconv.FormatUint(uint64(egid), 10) - ret.Thread.Capabilities.Permitted = ecsCapsFromU64(p.Creds.CapPermitted) - ret.Thread.Capabilities.Effective = ecsCapsFromU64(p.Creds.CapEffective) - - return ret -} - -func fillParent(process *types.Process, parent Process) { - reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(parent.Pids.StartTimeNs) - - interactive := interactiveFromTty(parent.CTty) - euid := parent.Creds.Euid - egid := parent.Creds.Egid - process.Parent.PID = parent.Pids.Tgid - process.Parent.Start = timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime) - process.Parent.Name = basename(parent.Filename) - process.Parent.Executable = parent.Filename - process.Parent.Args = parent.Argv - process.Parent.WorkingDirectory = parent.Cwd - process.Parent.Interactive = &interactive - process.Parent.User.ID = strconv.FormatUint(uint64(euid), 10) - process.Parent.Group.ID = strconv.FormatUint(uint64(egid), 10) -} - -func fillGroupLeader(process *types.Process, groupLeader Process) { - reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(groupLeader.Pids.StartTimeNs) - - interactive := interactiveFromTty(groupLeader.CTty) - euid := groupLeader.Creds.Euid - egid := groupLeader.Creds.Egid - process.GroupLeader.PID = groupLeader.Pids.Tgid - process.GroupLeader.Start = timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime) - process.GroupLeader.Name = basename(groupLeader.Filename) - process.GroupLeader.Executable = groupLeader.Filename - process.GroupLeader.Args = groupLeader.Argv - process.GroupLeader.WorkingDirectory = groupLeader.Cwd - process.GroupLeader.Interactive = &interactive - process.GroupLeader.User.ID = strconv.FormatUint(uint64(euid), 10) - process.GroupLeader.Group.ID = strconv.FormatUint(uint64(egid), 10) -} - -func fillSessionLeader(process *types.Process, sessionLeader Process) { - reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(sessionLeader.Pids.StartTimeNs) - - interactive := interactiveFromTty(sessionLeader.CTty) - euid := sessionLeader.Creds.Euid - egid := sessionLeader.Creds.Egid - process.SessionLeader.PID = sessionLeader.Pids.Tgid - process.SessionLeader.Start = timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime) - process.SessionLeader.Name = basename(sessionLeader.Filename) - process.SessionLeader.Executable = sessionLeader.Filename - process.SessionLeader.Args = sessionLeader.Argv - process.SessionLeader.WorkingDirectory = sessionLeader.Cwd - process.SessionLeader.Interactive = &interactive - process.SessionLeader.User.ID = strconv.FormatUint(uint64(euid), 10) - process.SessionLeader.Group.ID = strconv.FormatUint(uint64(egid), 10) -} - -func fillEntryLeader(process *types.Process, entryType EntryType, entryLeader Process) { - reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(entryLeader.Pids.StartTimeNs) - - interactive := interactiveFromTty(entryLeader.CTty) - euid := entryLeader.Creds.Euid - egid := entryLeader.Creds.Egid - process.EntryLeader.PID = entryLeader.Pids.Tgid - process.EntryLeader.Start = timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime) - process.EntryLeader.Name = basename(entryLeader.Filename) - process.EntryLeader.Executable = entryLeader.Filename - process.EntryLeader.Args = entryLeader.Argv - process.EntryLeader.WorkingDirectory = entryLeader.Cwd - process.EntryLeader.Interactive = &interactive - process.EntryLeader.User.ID = strconv.FormatUint(uint64(euid), 10) - process.EntryLeader.Group.ID = strconv.FormatUint(uint64(egid), 10) - - process.EntryLeader.EntryMeta.Type = string(entryType) -} - -func (db *SimpleDB) setEntityID(process *types.Process) { - if process.PID != 0 && process.Start != nil { - process.EntityID = db.calculateEntityIDv1(process.PID, *process.Start) - } - - if process.Parent.PID != 0 && process.Parent.Start != nil { - process.Parent.EntityID = db.calculateEntityIDv1(process.Parent.PID, *process.Parent.Start) - } - - if process.GroupLeader.PID != 0 && process.GroupLeader.Start != nil { - process.GroupLeader.EntityID = db.calculateEntityIDv1(process.GroupLeader.PID, *process.GroupLeader.Start) - } - - if process.SessionLeader.PID != 0 && process.SessionLeader.Start != nil { - process.SessionLeader.EntityID = db.calculateEntityIDv1(process.SessionLeader.PID, *process.SessionLeader.Start) - } - - if process.EntryLeader.PID != 0 && process.EntryLeader.Start != nil { - process.EntryLeader.EntityID = db.calculateEntityIDv1(process.EntryLeader.PID, *process.EntryLeader.Start) - } -} - -func setSameAsProcess(process *types.Process) { - if process.GroupLeader.PID != 0 && process.GroupLeader.Start != nil { - sameAsProcess := process.PID == process.GroupLeader.PID - process.GroupLeader.SameAsProcess = &sameAsProcess - } - - if process.SessionLeader.PID != 0 && process.SessionLeader.Start != nil { - sameAsProcess := process.PID == process.SessionLeader.PID - process.SessionLeader.SameAsProcess = &sameAsProcess - } - - if process.EntryLeader.PID != 0 && process.EntryLeader.Start != nil { - sameAsProcess := process.PID == process.EntryLeader.PID - process.EntryLeader.SameAsProcess = &sameAsProcess - } -} - -func (db *SimpleDB) GetProcess(pid uint32) (types.Process, error) { - db.RLock() - defer db.RUnlock() - - process, ok := db.processes[pid] - if !ok { - return types.Process{}, errors.New("process not found") - } - - ret := fullProcessFromDBProcess(process) - - if parent, ok := db.processes[process.Pids.Ppid]; ok { - fillParent(&ret, parent) - } - - if groupLeader, ok := db.processes[process.Pids.Pgid]; ok { - fillGroupLeader(&ret, groupLeader) - } - - if sessionLeader, ok := db.processes[process.Pids.Sid]; ok { - fillSessionLeader(&ret, sessionLeader) - } - - if entryLeaderPid, foundEntryLeaderPid := db.entryLeaderRelationships[process.Pids.Tgid]; foundEntryLeaderPid { - if entryLeader, foundEntryLeader := db.processes[entryLeaderPid]; foundEntryLeader { - // if there is an entry leader then there is a matching member in the entryLeaders table - fillEntryLeader(&ret, db.entryLeaders[entryLeaderPid], entryLeader) - } else { - db.logger.Errorf("failed to find entry leader entry %d for %d (%s)", entryLeaderPid, pid, db.processes[pid].Filename) - } - } else { - db.logger.Errorf("failed to find entry leader for %d (%s)", pid, db.processes[pid].Filename) - } - - db.setEntityID(&ret) - setSameAsProcess(&ret) - - return ret, nil -} - -func (db *SimpleDB) GetEntryType(pid uint32) (EntryType, error) { - db.RLock() - defer db.RUnlock() - - if entryType, ok := db.entryLeaders[pid]; ok { - return entryType, nil - } - return EntryUnknown, nil -} - -func (db *SimpleDB) ScrapeProcfs() []uint32 { - db.Lock() - defer db.Unlock() - - procs, err := db.procfs.GetAllProcesses() - if err != nil { - db.logger.Errorf("failed to get processes from procfs: %v", err) - return make([]uint32, 0) - } - - // sorting the slice to make sure that parents, session leaders, group - // leaders come first in the queue - sort.Slice(procs, func(i, j int) bool { - return procs[i].Pids.Tgid == procs[j].Pids.Ppid || - procs[i].Pids.Tgid == procs[j].Pids.Sid || - procs[i].Pids.Tgid == procs[j].Pids.Pgid - }) - - pids := make([]uint32, 0) - for _, procInfo := range procs { - process := Process{ - Pids: pidInfoFromProto(procInfo.Pids), - Creds: credInfoFromProto(procInfo.Creds), - CTty: ttyDevFromProto(procInfo.CTty), - Argv: procInfo.Argv, - Cwd: procInfo.Cwd, - Env: procInfo.Env, - Filename: procInfo.Filename, - } - - db.insertProcess(process) - pids = append(pids, process.Pids.Tgid) - } - - return pids -} diff --git a/x-pack/auditbeat/processors/add_session_metadata/processdb/simple_test.go b/x-pack/auditbeat/processors/add_session_metadata/processdb/simple_test.go deleted file mode 100644 index 41691e209462..000000000000 --- a/x-pack/auditbeat/processors/add_session_metadata/processdb/simple_test.go +++ /dev/null @@ -1,131 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package processdb - -import ( - "testing" - - "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/procfs" - "github.com/stretchr/testify/assert" - - "github.com/elastic/elastic-agent-libs/logp" - - "golang.org/x/sys/unix" -) - -var logger = logp.NewLogger("processdb") -var reader = procfs.NewMockReader() - -// glue function to fit the return type required by these tests -func newSimpleDBIntf(reader procfs.Reader) DB { - ret := NewSimpleDB(reader, *logger) - _ = ret.ScrapeProcfs() - return ret -} - -func TestSimpleSingleProcessSessionLeaderEntryTypeTerminal(t *testing.T) { - testSingleProcessSessionLeaderEntryTypeTerminal(newSimpleDBIntf)(t) -} - -func TestSimpleSingleProcessSessionLeaderLoginProcess(t *testing.T) { - testSingleProcessSessionLeaderLoginProcess(newSimpleDBIntf)(t) -} - -func TestSimpleSingleProcessSessionLeaderChildOfInit(t *testing.T) { - testSingleProcessSessionLeaderChildOfInit(newSimpleDBIntf)(t) -} - -func TestSimpleSingleProcessSessionLeaderChildOfSsmSessionWorker(t *testing.T) { - testSingleProcessSessionLeaderChildOfSsmSessionWorker(newSimpleDBIntf)(t) -} - -func TestSimpleSingleProcessSessionLeaderChildOfSshd(t *testing.T) { - testSingleProcessSessionLeaderChildOfSshd(newSimpleDBIntf)(t) -} - -func TestSimpleSingleProcessSessionLeaderChildOfContainerdShim(t *testing.T) { - testSingleProcessSessionLeaderChildOfContainerdShim(newSimpleDBIntf)(t) -} - -func TestSimpleSingleProcessSessionLeaderOfRunc(t *testing.T) { - testSingleProcessSessionLeaderChildOfRunc(newSimpleDBIntf)(t) -} - -func TestSimpleSingleProcessEmptyProcess(t *testing.T) { - testSingleProcessEmptyProcess(newSimpleDBIntf)(t) -} - -func TestSimpleSingleProcessOverwriteOldEntryLeader(t *testing.T) { - testSingleProcessOverwriteOldEntryLeader(newSimpleDBIntf)(t) -} - -func TestSimpleInitSshdBashLs(t *testing.T) { - testInitSshdBashLs(newSimpleDBIntf)(t) -} - -func TestSimpleInitSshdSshdBashLs(t *testing.T) { - testInitSshdSshdBashLs(newSimpleDBIntf)(t) -} - -func TestSimpleInitSshdSshdSshdBashLs(t *testing.T) { - testInitSshdSshdSshdBashLs(newSimpleDBIntf)(t) -} - -func TestSimpleInitContainerdContainerdShim(t *testing.T) { - testInitContainerdContainerdShim(newSimpleDBIntf)(t) -} - -func TestSimpleInitContainerdShimBashContainerdShimIsReparentedToInit(t *testing.T) { - testInitContainerdShimBashContainerdShimIsReparentedToInit(newSimpleDBIntf)(t) -} - -func TestSimpleInitContainerdShimPauseContainerdShimIsReparentedToInit(t *testing.T) { - testInitContainerdShimPauseContainerdShimIsReparentedToInit(newSimpleDBIntf)(t) -} - -func TestSimpleInitSshdBashLsAndGrepGrepOnlyHasGroupLeader(t *testing.T) { - testInitSshdBashLsAndGrepGrepOnlyHasGroupLeader(newSimpleDBIntf)(t) -} - -func TestSimpleInitSshdBashLsAndGrepGrepOnlyHasSessionLeader(t *testing.T) { - testInitSshdBashLsAndGrepGrepOnlyHasSessionLeader(newSimpleDBIntf)(t) -} - -func TestSimpleGrepInIsolation(t *testing.T) { - testGrepInIsolation(newSimpleDBIntf)(t) -} - -func TestSimpleKernelThreads(t *testing.T) { - testKernelThreads(newSimpleDBIntf)(t) -} - -func TestCapsFromU64ToECS(t *testing.T) { - expected := []string{"CAP_CHOWN"} - assert.Equal(t, expected, ecsCapsFromU64(uint64(1< Date: Wed, 17 Jan 2024 16:25:05 -0800 Subject: [PATCH 09/32] Pass DB by reference in tests --- .../add_session_metadata.go | 3 +++ .../add_session_metadata/processdb/db_test.go | 4 ++-- .../processdb/entry_leader_test.go | 24 ++++++++++++------- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go b/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go index 31aa881b6929..9ad2e721c904 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go +++ b/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go @@ -2,6 +2,9 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +//go:build linux +// +build linux + package add_session_metadata import ( diff --git a/x-pack/auditbeat/processors/add_session_metadata/processdb/db_test.go b/x-pack/auditbeat/processors/add_session_metadata/processdb/db_test.go index 2450eb7ee7be..59074b153b9b 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/processdb/db_test.go +++ b/x-pack/auditbeat/processors/add_session_metadata/processdb/db_test.go @@ -19,10 +19,10 @@ var logger = logp.NewLogger("processdb") var reader = procfs.NewMockReader() // glue function to fit the return type required by these tests -func newDBIntf(reader procfs.Reader) DB { +func newDBIntf(reader procfs.Reader) *DB { ret := NewDB(reader, *logger) _ = ret.ScrapeProcfs() - return *ret + return ret } diff --git a/x-pack/auditbeat/processors/add_session_metadata/processdb/entry_leader_test.go b/x-pack/auditbeat/processors/add_session_metadata/processdb/entry_leader_test.go index b8b5348df6a2..b6ae1b2d85d9 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/processdb/entry_leader_test.go +++ b/x-pack/auditbeat/processors/add_session_metadata/processdb/entry_leader_test.go @@ -12,11 +12,10 @@ import ( "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/procfs" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/types" - // "github.com/elastic/elastic-agent-libs/logp" ) type ( - createDBFn func(procfs.Reader) DB + createDBFn func(procfs.Reader) *DB testFn func(*testing.T) ) @@ -69,7 +68,8 @@ const ( // These tests should effectively serve as the spec for how we assign entry // leaders. When further entry meta types or cases are added, tests should be -func requireProcess(t *testing.T, db DB, pid uint32, processPath string) { +func requireProcess(t *testing.T, db *DB, pid uint32, processPath string) { + t.Helper() process, err := db.GetProcess(pid) require.Nil(t, err) require.Equal(t, pid, process.PID) @@ -81,19 +81,22 @@ func requireProcess(t *testing.T, db DB, pid uint32, processPath string) { } } -func requireParent(t *testing.T, db DB, pid uint32, ppid uint32) { +func requireParent(t *testing.T, db *DB, pid uint32, ppid uint32) { + t.Helper() process, err := db.GetProcess(pid) require.Nil(t, err) require.Equal(t, ppid, process.Parent.PID) } func requireParentUnset(t *testing.T, process types.Process) { + t.Helper() require.Equal(t, "", process.Parent.EntityID) require.Equal(t, uint32(0), process.Parent.PID) require.Nil(t, process.Parent.Start) } -func requireSessionLeader(t *testing.T, db DB, pid uint32, sid uint32) { +func requireSessionLeader(t *testing.T, db *DB, pid uint32, sid uint32) { + t.Helper() process, err := db.GetProcess(pid) require.Nil(t, err) require.Equal(t, sid, process.SessionLeader.PID) @@ -102,12 +105,14 @@ func requireSessionLeader(t *testing.T, db DB, pid uint32, sid uint32) { } func requireSessionLeaderUnset(t *testing.T, process types.Process) { + t.Helper() require.Equal(t, "", process.SessionLeader.EntityID) require.Equal(t, uint32(0), process.SessionLeader.PID) require.Nil(t, process.SessionLeader.Start) } -func requireGroupLeader(t *testing.T, db DB, pid uint32, pgid uint32) { +func requireGroupLeader(t *testing.T, db *DB, pid uint32, pgid uint32) { + t.Helper() process, err := db.GetProcess(pid) require.Nil(t, err) require.Equal(t, pgid, process.GroupLeader.PID) @@ -115,7 +120,8 @@ func requireGroupLeader(t *testing.T, db DB, pid uint32, pgid uint32) { require.Equal(t, pid == pgid, *process.GroupLeader.SameAsProcess) } -func requireEntryLeader(t *testing.T, db DB, pid uint32, entryPid uint32, expectedEntryType EntryType) { +func requireEntryLeader(t *testing.T, db *DB, pid uint32, entryPid uint32, expectedEntryType EntryType) { + t.Helper() process, err := db.GetProcess(pid) require.Nil(t, err) require.Equal(t, entryPid, process.EntryLeader.PID) @@ -128,13 +134,15 @@ func requireEntryLeader(t *testing.T, db DB, pid uint32, entryPid uint32, expect } func requireEntryLeaderUnset(t *testing.T, process types.Process) { + t.Helper() require.Equal(t, "", process.EntryLeader.EntityID) require.Equal(t, uint32(0), process.EntryLeader.PID) require.Nil(t, process.EntryLeader.Start) } // tries to construct fork event from what's in the db -func insertForkAndExec(t *testing.T, db DB, exec types.ProcessExecEvent) { +func insertForkAndExec(t *testing.T, db *DB, exec types.ProcessExecEvent) { + t.Helper() var fork types.ProcessForkEvent fork.ChildPids = exec.Pids parent, err := db.GetProcess(exec.Pids.Ppid) From 0cbb970952628177c85f0a2084732e4055abac39 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Wed, 17 Jan 2024 17:41:43 -0800 Subject: [PATCH 10/32] Rework entry leader tests --- .../add_session_metadata/processdb/db_test.go | 84 - .../processdb/entry_leader_test.go | 1797 ++++++++--------- 2 files changed, 886 insertions(+), 995 deletions(-) diff --git a/x-pack/auditbeat/processors/add_session_metadata/processdb/db_test.go b/x-pack/auditbeat/processors/add_session_metadata/processdb/db_test.go index 59074b153b9b..0ec8eba96e92 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/processdb/db_test.go +++ b/x-pack/auditbeat/processors/add_session_metadata/processdb/db_test.go @@ -18,14 +18,6 @@ import ( var logger = logp.NewLogger("processdb") var reader = procfs.NewMockReader() -// glue function to fit the return type required by these tests -func newDBIntf(reader procfs.Reader) *DB { - ret := NewDB(reader, *logger) - _ = ret.ScrapeProcfs() - return ret -} - - func TestGetTtyType(t *testing.T) { assert.Equal(t, TtyConsole, getTtyType(4, 0)) assert.Equal(t, Pts, getTtyType(136, 0)) @@ -33,82 +25,6 @@ func TestGetTtyType(t *testing.T) { assert.Equal(t, TtyUnknown, getTtyType(1000, 1000)) } -func TestSingleProcessSessionLeaderEntryTypeTerminal(t *testing.T) { - testSingleProcessSessionLeaderEntryTypeTerminal(newDBIntf)(t) -} - -func TestSingleProcessSessionLeaderLoginProcess(t *testing.T) { - testSingleProcessSessionLeaderLoginProcess(newDBIntf)(t) -} - -func TestSingleProcessSessionLeaderChildOfInit(t *testing.T) { - testSingleProcessSessionLeaderChildOfInit(newDBIntf)(t) -} - -func TestSingleProcessSessionLeaderChildOfSsmSessionWorker(t *testing.T) { - testSingleProcessSessionLeaderChildOfSsmSessionWorker(newDBIntf)(t) -} - -func TestSingleProcessSessionLeaderChildOfSshd(t *testing.T) { - testSingleProcessSessionLeaderChildOfSshd(newDBIntf)(t) -} - -func TestSingleProcessSessionLeaderChildOfContainerdShim(t *testing.T) { - testSingleProcessSessionLeaderChildOfContainerdShim(newDBIntf)(t) -} - -func TestSingleProcessSessionLeaderOfRunc(t *testing.T) { - testSingleProcessSessionLeaderChildOfRunc(newDBIntf)(t) -} - -func TestSingleProcessEmptyProcess(t *testing.T) { - testSingleProcessEmptyProcess(newDBIntf)(t) -} - -func TestSingleProcessOverwriteOldEntryLeader(t *testing.T) { - testSingleProcessOverwriteOldEntryLeader(newDBIntf)(t) -} - -func TestInitSshdBashLs(t *testing.T) { - testInitSshdBashLs(newDBIntf)(t) -} - -func TestInitSshdSshdBashLs(t *testing.T) { - testInitSshdSshdBashLs(newDBIntf)(t) -} - -func TestInitSshdSshdSshdBashLs(t *testing.T) { - testInitSshdSshdSshdBashLs(newDBIntf)(t) -} - -func TestInitContainerdContainerdShim(t *testing.T) { - testInitContainerdContainerdShim(newDBIntf)(t) -} - -func TestInitContainerdShimBashContainerdShimIsReparentedToInit(t *testing.T) { - testInitContainerdShimBashContainerdShimIsReparentedToInit(newDBIntf)(t) -} - -func TestInitContainerdShimPauseContainerdShimIsReparentedToInit(t *testing.T) { - testInitContainerdShimPauseContainerdShimIsReparentedToInit(newDBIntf)(t) -} - -func TestInitSshdBashLsAndGrepGrepOnlyHasGroupLeader(t *testing.T) { - testInitSshdBashLsAndGrepGrepOnlyHasGroupLeader(newDBIntf)(t) -} - -func TestInitSshdBashLsAndGrepGrepOnlyHasSessionLeader(t *testing.T) { - testInitSshdBashLsAndGrepGrepOnlyHasSessionLeader(newDBIntf)(t) -} - -func TestGrepInIsolation(t *testing.T) { - testGrepInIsolation(newDBIntf)(t) -} - -func TestKernelThreads(t *testing.T) { - testKernelThreads(newDBIntf)(t) -} - func TestCapsFromU64ToECS(t *testing.T) { expected := []string{"CAP_CHOWN"} assert.Equal(t, expected, ecsCapsFromU64(uint64(1<, ) // // Kernel threads should never have an entry meta type or entry leader set. -func testKernelThreads(createDB createDBFn) testFn { - return func(t *testing.T) { - reader := procfs.NewMockReader() - db := createDB(reader) - - kthreaddPid := uint32(2) - rcuGpPid := uint32(3) - - kthreaddPath := "kthreadd" - rcuGpPath := "rcu_gp" - - err := db.InsertExec(types.ProcessExecEvent{ - Filename: kthreaddPath, - Pids: types.PidInfo{ - Tgid: kthreaddPid, - Ppid: 1, - Sid: 0, - }, - }) - require.Nil(t, err) +func TestKernelThreads(t *testing.T) { + reader := procfs.NewMockReader() + db := NewDB(reader, *logger) - err = db.InsertExec(types.ProcessExecEvent{ - Filename: rcuGpPath, - Pids: types.PidInfo{ - Tgid: rcuGpPid, - Ppid: kthreaddPid, - Sid: 0, - }, - }) - require.Nil(t, err) + kthreaddPid := uint32(2) + rcuGpPid := uint32(3) - // kthreadd - kthreadd, err := db.GetProcess(kthreaddPid) - require.Nil(t, err) - requireParentUnset(t, kthreadd) - requireSessionLeaderUnset(t, kthreadd) - requireEntryLeaderUnset(t, kthreadd) + kthreaddPath := "kthreadd" + rcuGpPath := "rcu_gp" - requireProcess(t, db, kthreaddPid, kthreaddPath) + err := db.InsertExec(types.ProcessExecEvent{ + Filename: kthreaddPath, + Pids: types.PidInfo{ + Tgid: kthreaddPid, + Ppid: 1, + Sid: 0, + }, + }) + require.Nil(t, err) - // rcu_gp - rcuGp, err := db.GetProcess(rcuGpPid) - require.Nil(t, err) - requireSessionLeaderUnset(t, rcuGp) - requireEntryLeaderUnset(t, rcuGp) + err = db.InsertExec(types.ProcessExecEvent{ + Filename: rcuGpPath, + Pids: types.PidInfo{ + Tgid: rcuGpPid, + Ppid: kthreaddPid, + Sid: 0, + }, + }) + require.Nil(t, err) - requireProcess(t, db, rcuGpPid, rcuGpPath) - requireParent(t, db, rcuGpPid, kthreaddPid) - } + // kthreadd + kthreadd, err := db.GetProcess(kthreaddPid) + require.Nil(t, err) + requireParentUnset(t, kthreadd) + requireSessionLeaderUnset(t, kthreadd) + requireEntryLeaderUnset(t, kthreadd) + + requireProcess(t, db, kthreaddPid, kthreaddPath) + + // rcu_gp + rcuGp, err := db.GetProcess(rcuGpPid) + require.Nil(t, err) + requireSessionLeaderUnset(t, rcuGp) + requireEntryLeaderUnset(t, rcuGp) + + requireProcess(t, db, rcuGpPid, rcuGpPath) + requireParent(t, db, rcuGpPid, kthreaddPid) } From 06b706414d7c130de28aef67923adb4a1236fc5d Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Thu, 18 Jan 2024 10:06:17 -0800 Subject: [PATCH 11/32] Reformat processor --- .../add_session_metadata.go | 4 +- .../add_session_metadata_test.go | 53 ++++++++++--------- .../add_session_metadata/processdb/db.go | 42 +++++++-------- .../add_session_metadata/processdb/db_test.go | 3 +- .../processdb/entry_leader_test.go | 4 +- .../add_session_metadata/procfs/procfs.go | 1 + .../provider/ebpf_provider/ebpf_provider.go | 16 +++--- .../add_session_metadata/types/events.go | 10 ++-- 8 files changed, 68 insertions(+), 65 deletions(-) diff --git a/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go b/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go index 9ad2e721c904..68b98f4551c3 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go +++ b/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go @@ -76,8 +76,8 @@ func New(cfg *config.C) (beat.Processor, error) { } func (p *addSessionMetadata) Run(ev *beat.Event) (*beat.Event, error) { - _, err := ev.GetValue(p.config.PidField) - if err != nil { + _, err := ev.GetValue(p.config.PidField) + if err != nil { // Do not attempt to enrich events without PID; it's not a supported event return ev, nil } diff --git a/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata_test.go b/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata_test.go index 188e0c1d23b7..be51e8e2d3cc 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata_test.go +++ b/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata_test.go @@ -8,57 +8,58 @@ import ( "fmt" "testing" + "github.com/stretchr/testify/assert" + "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/processdb" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/procfs" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/types" - "github.com/stretchr/testify/assert" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" ) var ( - enrichTests = []struct{ - testName string + enrichTests = []struct { + testName string mockProcesses []types.ProcessExecEvent - config Config - input beat.Event - expected beat.Event - expect_error bool + config Config + input beat.Event + expected beat.Event + expect_error bool }{ { testName: "Enrich Process", config: Config{ ReplaceFields: false, - PidField: "process.pid", + PidField: "process.pid", }, mockProcesses: []types.ProcessExecEvent{ { Pids: types.PidInfo{ - Tid: uint32(100), + Tid: uint32(100), Tgid: uint32(100), Ppid: uint32(50), Pgid: uint32(100), - Sid: uint32(40), + Sid: uint32(40), }, - Cwd: "/", + Cwd: "/", Filename: "/bin/ls", }, { Pids: types.PidInfo{ - Tid: uint32(50), + Tid: uint32(50), Tgid: uint32(50), Ppid: uint32(40), - Sid: uint32(40), + Sid: uint32(40), }, }, { Pids: types.PidInfo{ - Tid: uint32(40), + Tid: uint32(40), Tgid: uint32(40), Ppid: uint32(1), - Sid: uint32(1), + Sid: uint32(1), }, }, }, @@ -72,9 +73,9 @@ var ( expected: beat.Event{ Fields: mapstr.M{ "process": mapstr.M{ - "executable": "/bin/ls", + "executable": "/bin/ls", "working_directory": "/", - "pid": uint32(100), + "pid": uint32(100), "parent": mapstr.M{ "pid": uint32(50), }, @@ -93,12 +94,12 @@ var ( testName: "No Pid Field in Event", config: Config{ ReplaceFields: false, - PidField: "process.pid", + PidField: "process.pid", }, input: beat.Event{ Fields: mapstr.M{ "process": mapstr.M{ - "executable": "ls", + "executable": "ls", "working_directory": "/", "parent": mapstr.M{ "pid": uint32(100), @@ -112,13 +113,13 @@ var ( testName: "Pid Not Number", config: Config{ ReplaceFields: false, - PidField: "process.pid", + PidField: "process.pid", }, input: beat.Event{ Fields: mapstr.M{ "process": mapstr.M{ - "pid": "xyz", - "executable": "ls", + "pid": "xyz", + "executable": "ls", "working_directory": "/", "parent": mapstr.M{ "pid": uint32(50), @@ -132,13 +133,13 @@ var ( testName: "PID not in DB", config: Config{ ReplaceFields: false, - PidField: "process.pid", + PidField: "process.pid", }, input: beat.Event{ Fields: mapstr.M{ "process": mapstr.M{ - "pid": "100", - "executable": "ls", + "pid": "100", + "executable": "ls", "working_directory": "/", "parent": mapstr.M{ "pid": uint32(100), @@ -164,7 +165,7 @@ func TestEnrich(t *testing.T) { } s := addSessionMetadata{ logger: logger, - db: db, + db: db, config: tt.config, } diff --git a/x-pack/auditbeat/processors/add_session_metadata/processdb/db.go b/x-pack/auditbeat/processors/add_session_metadata/processdb/db.go index 12992cd94bd7..36f610423458 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/processdb/db.go +++ b/x-pack/auditbeat/processors/add_session_metadata/processdb/db.go @@ -72,13 +72,13 @@ const ( ) type Process struct { - Pids types.PidInfo - Creds types.CredInfo - CTty types.TtyDev - Argv []string - Cwd string - Env map[string]string - Filename string + Pids types.PidInfo + Creds types.CredInfo + CTty types.TtyDev + Argv []string + Cwd string + Env map[string]string + Filename string } var ( @@ -314,13 +314,13 @@ func (db *DB) InsertExec(exec types.ProcessExecEvent) error { defer db.Unlock() proc := Process{ - Pids: pidInfoFromProto(exec.Pids), - Creds: credInfoFromProto(exec.Creds), - CTty: ttyDevFromProto(exec.CTty), - Argv: exec.Argv, - Cwd: exec.Cwd, - Env: exec.Env, - Filename: exec.Filename, + Pids: pidInfoFromProto(exec.Pids), + Creds: credInfoFromProto(exec.Creds), + CTty: ttyDevFromProto(exec.CTty), + Argv: exec.Argv, + Cwd: exec.Cwd, + Env: exec.Env, + Filename: exec.Filename, } db.processes[exec.Pids.Tgid] = proc @@ -686,13 +686,13 @@ func (db *DB) ScrapeProcfs() []uint32 { pids := make([]uint32, 0) for _, procInfo := range procs { process := Process{ - Pids: pidInfoFromProto(procInfo.Pids), - Creds: credInfoFromProto(procInfo.Creds), - CTty: ttyDevFromProto(procInfo.CTty), - Argv: procInfo.Argv, - Cwd: procInfo.Cwd, - Env: procInfo.Env, - Filename: procInfo.Filename, + Pids: pidInfoFromProto(procInfo.Pids), + Creds: credInfoFromProto(procInfo.Creds), + CTty: ttyDevFromProto(procInfo.CTty), + Argv: procInfo.Argv, + Cwd: procInfo.Cwd, + Env: procInfo.Env, + Filename: procInfo.Filename, } db.insertProcess(process) diff --git a/x-pack/auditbeat/processors/add_session_metadata/processdb/db_test.go b/x-pack/auditbeat/processors/add_session_metadata/processdb/db_test.go index 0ec8eba96e92..b41e81ae96c4 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/processdb/db_test.go +++ b/x-pack/auditbeat/processors/add_session_metadata/processdb/db_test.go @@ -7,9 +7,10 @@ package processdb import ( "testing" - "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/procfs" "github.com/stretchr/testify/assert" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/procfs" + "github.com/elastic/elastic-agent-libs/logp" "golang.org/x/sys/unix" diff --git a/x-pack/auditbeat/processors/add_session_metadata/processdb/entry_leader_test.go b/x-pack/auditbeat/processors/add_session_metadata/processdb/entry_leader_test.go index f513ef5e6d74..7b992790cffc 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/processdb/entry_leader_test.go +++ b/x-pack/auditbeat/processors/add_session_metadata/processdb/entry_leader_test.go @@ -177,10 +177,10 @@ var systemdPath = "/sbin/systemd" func populateProcfsWithInit(reader *procfs.MockReader) { reader.AddEntry(1, procfs.ProcessInfo{ Pids: types.PidInfo{ - Tid: 1, + Tid: 1, Tgid: 1, Pgid: 0, - Sid: 1, + Sid: 1, }, Filename: systemdPath, }) diff --git a/x-pack/auditbeat/processors/add_session_metadata/procfs/procfs.go b/x-pack/auditbeat/processors/add_session_metadata/procfs/procfs.go index 2496479abe1c..47ea6bb775f5 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/procfs/procfs.go +++ b/x-pack/auditbeat/processors/add_session_metadata/procfs/procfs.go @@ -206,6 +206,7 @@ func (r ProcfsReader) GetProcess(pid uint32) (ProcessInfo, error) { } return r.getProcessInfo(proc) } + // returns empty slice on error func (r ProcfsReader) GetAllProcesses() ([]ProcessInfo, error) { procs, err := procfs.AllProcs() diff --git a/x-pack/auditbeat/processors/add_session_metadata/provider/ebpf_provider/ebpf_provider.go b/x-pack/auditbeat/processors/add_session_metadata/provider/ebpf_provider/ebpf_provider.go index f1f501f853e6..9b94d88d7fdd 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/provider/ebpf_provider/ebpf_provider.go +++ b/x-pack/auditbeat/processors/add_session_metadata/provider/ebpf_provider/ebpf_provider.go @@ -21,9 +21,9 @@ import ( ) type prvdr struct { - ctx context.Context - logger *logp.Logger - db *processdb.DB + ctx context.Context + logger *logp.Logger + db *processdb.DB } func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (provider.Provider, error) { @@ -120,10 +120,10 @@ func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (pr Major: body.CTTY.Major, Minor: body.CTTY.Minor, }, - Cwd: body.Cwd, - Argv: deepcopy.Copy(body.Argv).([]string), - Env: deepcopy.Copy(body.Env).(map[string]string), - Filename: body.Filename, + Cwd: body.Cwd, + Argv: deepcopy.Copy(body.Argv).([]string), + Env: deepcopy.Copy(body.Env).(map[string]string), + Filename: body.Filename, } p.db.InsertExec(pe) case ebpfevents.EventTypeProcessExit: @@ -141,7 +141,7 @@ func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (pr Sid: body.Pids.Sid, StartTimeNs: body.Pids.StartTimeNs, }, - ExitCode: body.ExitCode, + ExitCode: body.ExitCode, } p.db.InsertExit(pe) } diff --git a/x-pack/auditbeat/processors/add_session_metadata/types/events.go b/x-pack/auditbeat/processors/add_session_metadata/types/events.go index 68dd53aa1d97..dd3daaabc988 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/types/events.go +++ b/x-pack/auditbeat/processors/add_session_metadata/types/events.go @@ -16,7 +16,7 @@ const ( ) type ( - Field uint32 + Field uint32 ) const ( @@ -78,10 +78,10 @@ type ProcessExecEvent struct { CTty TtyDev // varlen fields - Cwd string - Argv []string - Env map[string]string - Filename string + Cwd string + Argv []string + Env map[string]string + Filename string } type ProcessExitEvent struct { From be57ad8316e570f635b007507f64d2170e0f5661 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Thu, 18 Jan 2024 10:11:39 -0800 Subject: [PATCH 12/32] Only build tests on Linux --- .../processors/add_session_metadata/add_session_metadata.go | 1 - .../add_session_metadata/add_session_metadata_test.go | 2 ++ .../processors/add_session_metadata/processdb/db_test.go | 2 ++ .../add_session_metadata/processdb/entry_leader_test.go | 2 ++ .../processors/add_session_metadata/timeutils/time_test.go | 2 ++ 5 files changed, 8 insertions(+), 1 deletion(-) diff --git a/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go b/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go index 68b98f4551c3..8dfa9d6d8d73 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go +++ b/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go @@ -3,7 +3,6 @@ // you may not use this file except in compliance with the Elastic License. //go:build linux -// +build linux package add_session_metadata diff --git a/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata_test.go b/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata_test.go index be51e8e2d3cc..e6d247c81b8d 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata_test.go +++ b/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata_test.go @@ -2,6 +2,8 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +//go:build linux + package add_session_metadata import ( diff --git a/x-pack/auditbeat/processors/add_session_metadata/processdb/db_test.go b/x-pack/auditbeat/processors/add_session_metadata/processdb/db_test.go index b41e81ae96c4..a8ff18ba32ca 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/processdb/db_test.go +++ b/x-pack/auditbeat/processors/add_session_metadata/processdb/db_test.go @@ -2,6 +2,8 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +//go:build linux + package processdb import ( diff --git a/x-pack/auditbeat/processors/add_session_metadata/processdb/entry_leader_test.go b/x-pack/auditbeat/processors/add_session_metadata/processdb/entry_leader_test.go index 7b992790cffc..f06e7422b414 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/processdb/entry_leader_test.go +++ b/x-pack/auditbeat/processors/add_session_metadata/processdb/entry_leader_test.go @@ -2,6 +2,8 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +//go:build linux + package processdb import ( diff --git a/x-pack/auditbeat/processors/add_session_metadata/timeutils/time_test.go b/x-pack/auditbeat/processors/add_session_metadata/timeutils/time_test.go index 94557e160274..c679e7fb39c9 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/timeutils/time_test.go +++ b/x-pack/auditbeat/processors/add_session_metadata/timeutils/time_test.go @@ -2,6 +2,8 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +//go:build linux + package timeutils import ( From 9ad78110006e1092bbb95b835fbcb83165ceb080 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Thu, 18 Jan 2024 13:02:25 -0800 Subject: [PATCH 13/32] Add linux build directive to all files in processor --- x-pack/auditbeat/processors/add_session_metadata/config.go | 2 ++ .../auditbeat/processors/add_session_metadata/processdb/db.go | 2 ++ x-pack/auditbeat/processors/add_session_metadata/procfs/mock.go | 2 ++ .../auditbeat/processors/add_session_metadata/procfs/procfs.go | 2 ++ .../provider/ebpf_provider/ebpf_provider.go | 2 ++ .../processors/add_session_metadata/provider/provider.go | 2 ++ .../auditbeat/processors/add_session_metadata/timeutils/time.go | 2 ++ 7 files changed, 14 insertions(+) diff --git a/x-pack/auditbeat/processors/add_session_metadata/config.go b/x-pack/auditbeat/processors/add_session_metadata/config.go index 8a8cfc75e6d7..f2134edb57d8 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/config.go +++ b/x-pack/auditbeat/processors/add_session_metadata/config.go @@ -2,6 +2,8 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +//go:build linux + package add_session_metadata // Config for add_session_metadata processor. diff --git a/x-pack/auditbeat/processors/add_session_metadata/processdb/db.go b/x-pack/auditbeat/processors/add_session_metadata/processdb/db.go index 36f610423458..84d8b61fa7cc 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/processdb/db.go +++ b/x-pack/auditbeat/processors/add_session_metadata/processdb/db.go @@ -2,6 +2,8 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +//go:build linux + package processdb import ( diff --git a/x-pack/auditbeat/processors/add_session_metadata/procfs/mock.go b/x-pack/auditbeat/processors/add_session_metadata/procfs/mock.go index bf3120add8d5..1689873044ec 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/procfs/mock.go +++ b/x-pack/auditbeat/processors/add_session_metadata/procfs/mock.go @@ -2,6 +2,8 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +//go:build linux + package procfs import ( diff --git a/x-pack/auditbeat/processors/add_session_metadata/procfs/procfs.go b/x-pack/auditbeat/processors/add_session_metadata/procfs/procfs.go index 47ea6bb775f5..647986c23bfe 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/procfs/procfs.go +++ b/x-pack/auditbeat/processors/add_session_metadata/procfs/procfs.go @@ -2,6 +2,8 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +//go:build linux + package procfs import ( diff --git a/x-pack/auditbeat/processors/add_session_metadata/provider/ebpf_provider/ebpf_provider.go b/x-pack/auditbeat/processors/add_session_metadata/provider/ebpf_provider/ebpf_provider.go index 9b94d88d7fdd..85ece4b0ad85 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/provider/ebpf_provider/ebpf_provider.go +++ b/x-pack/auditbeat/processors/add_session_metadata/provider/ebpf_provider/ebpf_provider.go @@ -2,6 +2,8 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +//go:build linux + package ebpf_provider import ( diff --git a/x-pack/auditbeat/processors/add_session_metadata/provider/provider.go b/x-pack/auditbeat/processors/add_session_metadata/provider/provider.go index c651c1fcffb4..e3fa1547806c 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/provider/provider.go +++ b/x-pack/auditbeat/processors/add_session_metadata/provider/provider.go @@ -2,6 +2,8 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +//go:build linux + package provider import ( diff --git a/x-pack/auditbeat/processors/add_session_metadata/timeutils/time.go b/x-pack/auditbeat/processors/add_session_metadata/timeutils/time.go index daa733349f54..fc808db5cdb5 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/timeutils/time.go +++ b/x-pack/auditbeat/processors/add_session_metadata/timeutils/time.go @@ -2,6 +2,8 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +//go:build linux + package timeutils import ( From 3598b3ca672c57ecd058307ad9d903c0ff8c2099 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Mon, 22 Jan 2024 15:18:58 -0800 Subject: [PATCH 14/32] Changes for PR feedback * Changed to use time.Duration in timeutils for process start NS * Used go-cmp library to compare ECS docs in tests --- .../add_session_metadata_test.go | 119 ++++++++++++++---- .../add_session_metadata/processdb/db.go | 11 +- .../add_session_metadata/timeutils/time.go | 8 +- .../timeutils/time_test.go | 10 +- 4 files changed, 107 insertions(+), 41 deletions(-) diff --git a/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata_test.go b/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata_test.go index e6d247c81b8d..df51a232591e 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata_test.go +++ b/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata_test.go @@ -7,9 +7,9 @@ package add_session_metadata import ( - "fmt" "testing" + "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/assert" "github.com/elastic/beats/v7/libbeat/beat" @@ -153,6 +153,76 @@ var ( }, } + filterTests = []struct { + testName string + mx mapstr.M + my mapstr.M + expected bool + }{ + { + testName: "equal", + mx: mapstr.M{ + "key1": "A", + "key2": mapstr.M{ + "key2_2": 2.0, + }, + "key3": 1, + }, + my: mapstr.M{ + "key1": "A", + "key2": mapstr.M{ + "key2_2": 2.0, + }, + "key3": 1, + }, + expected: true, + }, + { + testName: "mismatched values", + mx: mapstr.M{ + "key1": "A", + "key2": "B", + "key3": "C", + }, + my: mapstr.M{ + "key1": "A", + "key2": "X", + "key3": "C", + }, + expected: false, + }, + { + testName: "ignore key only in 2nd map", + mx: mapstr.M{ + "key1": "A", + "key2": "B", + }, + my: mapstr.M{ + "key1": "A", + "key2": "B", + "key3": "C", + }, + expected: true, + }, + { + testName: "nested mismatch", + mx: mapstr.M{ + "key1": "A", + "key2": mapstr.M{ + "key2_2": "B", + }, + }, + my: mapstr.M{ + "key1": "A", + "key2": mapstr.M{ + "key2_2": 2.0, + }, + "key3": 1, + }, + expected: false, + }, + } + logger = logp.NewLogger("add_session_metadata_test") ) @@ -177,38 +247,33 @@ func TestEnrich(t *testing.T) { } else { assert.Nil(t, err, "%s: enrich error: %w", tt.testName, err) assert.NotNil(t, actual, "%s: returned nil event", tt.testName) + //Validate output - eq, msg := compareMapstr(tt.expected.Fields, actual.Fields) - assert.True(t, eq, "%s: actual does not match expected: %s\n\nactual: \"%v\"\n\nexpected: \"%v\"", tt.testName, msg, actual, tt.expected) + if diff := cmp.Diff(tt.expected.Fields, actual.Fields, ignoreMissingFrom(tt.expected.Fields)); diff != "" { + t.Errorf("field mismatch:\n%s", diff) + } } } } -// compareMapstr will compare that all fields in `a` have equal value to the fields in `b`. -// Note: Only fields that exist in `a` are compared; if `b` has additional fields, they do not affect the comparison -func compareMapstr(a mapstr.M, b mapstr.M) (equal bool, msg string) { - equal = false - msg = "" - - aFlat := a.Flatten() - bFlat := b.Flatten() +// IgnoreMissingFrom returns a filter that will ignore all fields missing from m +func ignoreMissingFrom(m mapstr.M) cmp.Option { + return cmp.FilterPath(func(p cmp.Path) bool { + mi, ok := p.Index(-1).(cmp.MapIndex) + if !ok { + return false + } + vx, _ := mi.Values() + return !vx.IsValid() + }, cmp.Ignore()) +} - for _, key := range *a.FlattenKeys() { - valA, err := aFlat.GetValue(key) - if err == nil { - // FlattenKeys returns inner and leaf nodes, we only need to consider leaf nodes - // GetValue will return error when attempting to read inner nodes; these keys are ignored - valB, err := bFlat.GetValue(key) - if err != nil { - msg = fmt.Sprintf("%s not found in mapstr b", key) - return - } - if valA != valB { - msg = fmt.Sprintf("mismatch in key %s: \"%v\" \"%v\"", key, valA, valB) - return - } +// TestFilter ensures `ignoreMissingFrom` filter is working as expected +// Note: This validates test code only +func TestFilter(t *testing.T) { + for _, tt := range filterTests { + if eq := cmp.Equal(tt.mx, tt.my, ignoreMissingFrom(tt.mx)); eq != tt.expected { + t.Errorf("%s: unexpected comparator result", tt.testName) } } - equal = true - return } diff --git a/x-pack/auditbeat/processors/add_session_metadata/processdb/db.go b/x-pack/auditbeat/processors/add_session_metadata/processdb/db.go index 84d8b61fa7cc..818e01be8208 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/processdb/db.go +++ b/x-pack/auditbeat/processors/add_session_metadata/processdb/db.go @@ -362,19 +362,20 @@ func (db *DB) evaluateEntryLeader(p Process) *uint32 { ttyType := getTtyType(p.CTty.Major, p.CTty.Minor) procBasename := basename(p.Filename) - if ttyType == Tty { + switch { + case ttyType == Tty: db.createEntryLeader(pid, Terminal) db.logger.Debugf("entry_eval %d: entry type is terminal", p.Pids.Tgid) return &pid - } else if ttyType == TtyConsole && procBasename == "login" { + case ttyType == TtyConsole && procBasename == "login": db.createEntryLeader(pid, EntryConsole) db.logger.Debugf("entry_eval %d: entry type is console", p.Pids.Tgid) return &pid - } else if p.Pids.Ppid == 1 { + case p.Pids.Ppid == 1: db.createEntryLeader(pid, Init) db.logger.Debugf("entry_eval %d: entry type is init", p.Pids.Tgid) return &pid - } else if !isFilteredExecutable(procBasename) { + case !isFilteredExecutable(procBasename): if parent, ok := db.processes[p.Pids.Ppid]; ok { parentBasename := basename(parent.Filename) if ttyType == Pts && parentBasename == "ssm-session-worker" { @@ -392,7 +393,7 @@ func (db *DB) evaluateEntryLeader(p Process) *uint32 { return &pid } } - } else { + default: db.logger.Debugf("entry_eval %d: is a filtered executable: %s", p.Pids.Tgid, procBasename) } } diff --git a/x-pack/auditbeat/processors/add_session_metadata/timeutils/time.go b/x-pack/auditbeat/processors/add_session_metadata/timeutils/time.go index fc808db5cdb5..257d49f4dc53 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/timeutils/time.go +++ b/x-pack/auditbeat/processors/add_session_metadata/timeutils/time.go @@ -44,8 +44,8 @@ func TicksToNs(ticks uint64) uint64 { return ticks * uint64(time.Second.Nanoseconds()) / ticksPerSecond } -func TimeFromNsSinceBoot(ns uint64) *time.Time { - timestamp := bootTime.Add(time.Duration(ns)) +func TimeFromNsSinceBoot(t time.Duration) *time.Time { + timestamp := bootTime.Add(t) return ×tamp } @@ -59,6 +59,6 @@ func TimeFromNsSinceBoot(ns uint64) *time.Time { // get this precision from `sysconf()` // - We store timestamps as nanoseconds, but reduce the precision to 1/100th // second -func ReduceTimestampPrecision(timeNs uint64) uint64 { - return timeNs - (timeNs % (uint64(time.Second.Nanoseconds()) / ticksPerSecond)) +func ReduceTimestampPrecision(timeNs uint64) time.Duration { + return time.Duration(timeNs).Truncate(time.Second / time.Duration(ticksPerSecond)) } diff --git a/x-pack/auditbeat/processors/add_session_metadata/timeutils/time_test.go b/x-pack/auditbeat/processors/add_session_metadata/timeutils/time_test.go index c679e7fb39c9..1aa5abdf469a 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/timeutils/time_test.go +++ b/x-pack/auditbeat/processors/add_session_metadata/timeutils/time_test.go @@ -14,11 +14,11 @@ import ( ) func TestReduceTimestampPrecision(t *testing.T) { - oneSecond := uint64(time.Second.Nanoseconds()) - result1 := ReduceTimestampPrecision(oneSecond) - require.Equal(t, oneSecond, result1) + oneSecond := time.Second.Nanoseconds() + result1 := ReduceTimestampPrecision(uint64(oneSecond)) + require.Equal(t, time.Duration(oneSecond), result1) oneSecondWithDelay := oneSecond + 10 - result2 := ReduceTimestampPrecision(oneSecondWithDelay) - require.Equal(t, oneSecond, result2) + result2 := ReduceTimestampPrecision(uint64(oneSecondWithDelay)) + require.Equal(t, time.Duration(oneSecond), result2) } From 17fdf1c82a86fa11a0d80c675beec66c67226c13 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Mon, 22 Jan 2024 15:43:26 -0800 Subject: [PATCH 15/32] Add empty file to add_session_metadata package --- x-pack/auditbeat/processors/add_session_metadata/empty.go | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 x-pack/auditbeat/processors/add_session_metadata/empty.go diff --git a/x-pack/auditbeat/processors/add_session_metadata/empty.go b/x-pack/auditbeat/processors/add_session_metadata/empty.go new file mode 100644 index 000000000000..2454fb84c137 --- /dev/null +++ b/x-pack/auditbeat/processors/add_session_metadata/empty.go @@ -0,0 +1,5 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package add_session_metadata From 1ab47523c40d68f2804651adf29141a92f97efb9 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Tue, 23 Jan 2024 12:28:34 -0800 Subject: [PATCH 16/32] Fix linter warnings Fix linter warnings and upgrade go-libaudit to v2.5.0 --- NOTICE.txt | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- .../add_session_metadata/add_session_metadata.go | 7 +++++-- .../add_session_metadata_test.go | 4 +++- .../add_session_metadata/processdb/db_test.go | 3 --- .../provider/ebpf_provider/ebpf_provider.go | 15 ++++++++++++--- 7 files changed, 25 insertions(+), 14 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index 783237c21390..f8dbafc4959f 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -13674,11 +13674,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-elasticsearc -------------------------------------------------------------------------------- Dependency : github.com/elastic/go-libaudit/v2 -Version: v2.4.0 +Version: v2.5.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/go-libaudit/v2@v2.4.0/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/elastic/go-libaudit/v2@v2.5.0/LICENSE.txt: Apache License diff --git a/go.mod b/go.mod index 828a66be9aa6..7c78327ea0de 100644 --- a/go.mod +++ b/go.mod @@ -71,7 +71,7 @@ require ( github.com/eclipse/paho.mqtt.golang v1.3.5 github.com/elastic/elastic-agent-client/v7 v7.6.0 github.com/elastic/go-concert v0.2.0 - github.com/elastic/go-libaudit/v2 v2.4.0 + github.com/elastic/go-libaudit/v2 v2.5.0 github.com/elastic/go-licenser v0.4.1 github.com/elastic/go-lookslike v1.0.1 github.com/elastic/go-lumber v0.1.2-0.20220819171948-335fde24ea0f diff --git a/go.sum b/go.sum index 42e9eb198a96..939923a6e2e4 100644 --- a/go.sum +++ b/go.sum @@ -681,8 +681,8 @@ github.com/elastic/go-concert v0.2.0 h1:GAQrhRVXprnNjtvTP9pWJ1d4ToEA4cU5ci7TwTa2 github.com/elastic/go-concert v0.2.0/go.mod h1:HWjpO3IAEJUxOeaJOWXWEp7imKd27foxz9V5vegC/38= github.com/elastic/go-elasticsearch/v8 v8.11.1 h1:1VgTgUTbpqQZ4uE+cPjkOvy/8aw1ZvKcU0ZUE5Cn1mc= github.com/elastic/go-elasticsearch/v8 v8.11.1/go.mod h1:GU1BJHO7WeamP7UhuElYwzzHtvf9SDmeVpSSy9+o6Qg= -github.com/elastic/go-libaudit/v2 v2.4.0 h1:PqaGnB+dncrdUXqzQMyJu/dGysAtk6m5V3GIBMY473I= -github.com/elastic/go-libaudit/v2 v2.4.0/go.mod h1:AjlnhinP+kKQuUJoXLVrqxBM8uyhQmkzoV6jjsCFP4Q= +github.com/elastic/go-libaudit/v2 v2.5.0 h1:5OK919QRnGtcjVBz3n/cs5F42im1mPlVTA9TyIn2K54= +github.com/elastic/go-libaudit/v2 v2.5.0/go.mod h1:AjlnhinP+kKQuUJoXLVrqxBM8uyhQmkzoV6jjsCFP4Q= github.com/elastic/go-licenser v0.4.1 h1:1xDURsc8pL5zYT9R29425J3vkHdt4RT5TNEMeRN48x4= github.com/elastic/go-licenser v0.4.1/go.mod h1:V56wHMpmdURfibNBggaSBfqgPxyT1Tldns1i87iTEvU= github.com/elastic/go-lookslike v1.0.1 h1:qVieyn6i/kx4xntar1cEB0qrGHVGNCX5KC8czAaTW/0= diff --git a/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go b/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go index 8dfa9d6d8d73..05b97fca295a 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go +++ b/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go @@ -78,7 +78,7 @@ func (p *addSessionMetadata) Run(ev *beat.Event) (*beat.Event, error) { _, err := ev.GetValue(p.config.PidField) if err != nil { // Do not attempt to enrich events without PID; it's not a supported event - return ev, nil + return ev, nil //nolint:nilerr // Running on events without PID is expected } err = p.provider.UpdateDB(ev) @@ -117,7 +117,10 @@ func (p *addSessionMetadata) enrich(ev *beat.Event) (*beat.Event, error) { processMap := fullProcess.ToMap() - mapstr.MergeFieldsDeep(result.Fields["process"].(mapstr.M), processMap, true) + err = mapstr.MergeFieldsDeep(result.Fields["process"].(mapstr.M), processMap, true) + if err != nil { + return nil, fmt.Errorf("merging enriched fields with event: %w", err) + } if p.config.ReplaceFields { if err := p.replaceFields(result); err != nil { diff --git a/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata_test.go b/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata_test.go index df51a232591e..8135ce9f9f6c 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata_test.go +++ b/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata_test.go @@ -241,7 +241,9 @@ func TestEnrich(t *testing.T) { config: tt.config, } - actual, err := s.enrich(&tt.input) + // avoid taking address of loop variable + i := tt.input + actual, err := s.enrich(&i) if tt.expect_error { assert.Error(t, err, "%s: error unexpectedly nil", tt.testName) } else { diff --git a/x-pack/auditbeat/processors/add_session_metadata/processdb/db_test.go b/x-pack/auditbeat/processors/add_session_metadata/processdb/db_test.go index a8ff18ba32ca..a6c041710fc6 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/processdb/db_test.go +++ b/x-pack/auditbeat/processors/add_session_metadata/processdb/db_test.go @@ -11,15 +11,12 @@ import ( "github.com/stretchr/testify/assert" - "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/procfs" - "github.com/elastic/elastic-agent-libs/logp" "golang.org/x/sys/unix" ) var logger = logp.NewLogger("processdb") -var reader = procfs.NewMockReader() func TestGetTtyType(t *testing.T) { assert.Equal(t, TtyConsole, getTtyType(4, 0)) diff --git a/x-pack/auditbeat/processors/add_session_metadata/provider/ebpf_provider/ebpf_provider.go b/x-pack/auditbeat/processors/add_session_metadata/provider/ebpf_provider/ebpf_provider.go index 85ece4b0ad85..1b3e39b0c87b 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/provider/ebpf_provider/ebpf_provider.go +++ b/x-pack/auditbeat/processors/add_session_metadata/provider/ebpf_provider/ebpf_provider.go @@ -92,7 +92,10 @@ func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (pr CapEffective: body.Creds.CapEffective, }, } - p.db.InsertFork(pe) + if err := p.db.InsertFork(pe); err != nil { + p.logger.Errorf("insert fork: %w", err) + continue + } case ebpfevents.EventTypeProcessExec: body, ok := ev.Body.(*ebpfevents.ProcessExec) if !ok { @@ -127,7 +130,10 @@ func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (pr Env: deepcopy.Copy(body.Env).(map[string]string), Filename: body.Filename, } - p.db.InsertExec(pe) + if err := p.db.InsertExec(pe); err != nil { + p.logger.Errorf("insert exec: %w", err) + continue + } case ebpfevents.EventTypeProcessExit: body, ok := ev.Body.(*ebpfevents.ProcessExit) if !ok { @@ -145,7 +151,10 @@ func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (pr }, ExitCode: body.ExitCode, } - p.db.InsertExit(pe) + if err := p.db.InsertExit(pe); err != nil { + p.logger.Errorf("insert exit: %w", err) + continue + } } continue } From 145f627f72977a2159d7ce98d7e8091196590003 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Wed, 24 Jan 2024 12:36:00 -0800 Subject: [PATCH 17/32] Use single channel from epbevents ebpfevents library has been updated to use a single channel. Updated to use latest ebpfevents library and the single channel. --- NOTICE.txt | 4 +- go.mod | 2 +- go.sum | 4 +- .../provider/ebpf_provider/ebpf_provider.go | 211 +++++++++--------- 4 files changed, 109 insertions(+), 112 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index f8dbafc4959f..b0b06b031e7b 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -12257,11 +12257,11 @@ SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/elastic/ebpfevents -Version: v0.1.0 +Version: v0.3.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/ebpfevents@v0.1.0/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/elastic/ebpfevents@v0.3.0/LICENSE.txt: The https://github.com/elastic/ebpfevents repository contains source code under various licenses: diff --git a/go.mod b/go.mod index 7c78327ea0de..6456d66f747d 100644 --- a/go.mod +++ b/go.mod @@ -200,7 +200,7 @@ require ( github.com/aws/smithy-go v1.13.5 github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5 github.com/elastic/bayeux v1.0.5 - github.com/elastic/ebpfevents v0.1.0 + github.com/elastic/ebpfevents v0.3.0 github.com/elastic/elastic-agent-autodiscover v0.6.7 github.com/elastic/elastic-agent-libs v0.7.3 github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 diff --git a/go.sum b/go.sum index 939923a6e2e4..149598ca524d 100644 --- a/go.sum +++ b/go.sum @@ -660,8 +660,8 @@ github.com/elastic/bayeux v1.0.5 h1:UceFq01ipmT3S8DzFK+uVAkbCdiPR0Bqei8qIGmUeY0= github.com/elastic/bayeux v1.0.5/go.mod h1:CSI4iP7qeo5MMlkznGvYKftp8M7qqP/3nzmVZoXHY68= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY= -github.com/elastic/ebpfevents v0.1.0 h1:Kr62fVcDSrPYpwsW3FXUOcmImHqbiRfwmb6fZOw5PgI= -github.com/elastic/ebpfevents v0.1.0/go.mod h1:o21z5xup/9dK8u0Hg9bZRflSqqj1Zu5h2dg2hSTcUPQ= +github.com/elastic/ebpfevents v0.3.0 h1:tN2X+FNyV2o1x81tX1anAyVwXOTd1iRNxlkhf6zVY24= +github.com/elastic/ebpfevents v0.3.0/go.mod h1:o21z5xup/9dK8u0Hg9bZRflSqqj1Zu5h2dg2hSTcUPQ= github.com/elastic/elastic-agent-autodiscover v0.6.7 h1:+KVjltN0rPsBrU8b156gV4lOTBgG/vt0efFCFARrf3g= github.com/elastic/elastic-agent-autodiscover v0.6.7/go.mod h1:hFeFqneS2r4jD0/QzGkrNk0YVdN0JGh7lCWdsH7zcI4= github.com/elastic/elastic-agent-client/v7 v7.6.0 h1:FEn6FjzynW4TIQo5G096Tr7xYK/P5LY9cSS6wRbXZTc= diff --git a/x-pack/auditbeat/processors/add_session_metadata/provider/ebpf_provider/ebpf_provider.go b/x-pack/auditbeat/processors/add_session_metadata/provider/ebpf_provider/ebpf_provider.go index 1b3e39b0c87b..69f28df20f35 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/provider/ebpf_provider/ebpf_provider.go +++ b/x-pack/auditbeat/processors/add_session_metadata/provider/ebpf_provider/ebpf_provider.go @@ -41,122 +41,119 @@ func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (pr return prvdr{}, err } - events := make(chan ebpfevents.Event) - errors := make(chan error) + records := make(chan ebpfevents.Record) - go l.EventLoop(p.ctx, events, errors) + go l.EventLoop(p.ctx, records) go func(logger logp.Logger) { for { - select { - case err := <-errors: + r := <-records + if r.Error != nil { logger.Errorf("recv'd error: %w", err) continue - case ev := <-events: - if err != nil { - logger.Errorf("marshal event: %w", err) + } + if r.Event == nil { + continue + } + ev := r.Event + switch ev.Type { + case ebpfevents.EventTypeProcessFork: + body, ok := ev.Body.(*ebpfevents.ProcessFork) + if !ok { + logger.Errorf("unexpected event body") continue } - switch ev.Type { - case ebpfevents.EventTypeProcessFork: - body, ok := ev.Body.(*ebpfevents.ProcessFork) - if !ok { - logger.Errorf("unexpected event body") - continue - } - pe := types.ProcessForkEvent{ - ParentPids: types.PidInfo{ - Tid: body.ParentPids.Tid, - Tgid: body.ParentPids.Tgid, - Ppid: body.ParentPids.Ppid, - Pgid: body.ParentPids.Pgid, - Sid: body.ParentPids.Sid, - StartTimeNs: body.ParentPids.StartTimeNs, - }, - ChildPids: types.PidInfo{ - Tid: body.ChildPids.Tid, - Tgid: body.ChildPids.Tgid, - Ppid: body.ChildPids.Ppid, - Pgid: body.ChildPids.Pgid, - Sid: body.ChildPids.Sid, - StartTimeNs: body.ChildPids.StartTimeNs, - }, - Creds: types.CredInfo{ - Ruid: body.Creds.Ruid, - Rgid: body.Creds.Rgid, - Euid: body.Creds.Euid, - Egid: body.Creds.Egid, - Suid: body.Creds.Suid, - Sgid: body.Creds.Sgid, - CapPermitted: body.Creds.CapPermitted, - CapEffective: body.Creds.CapEffective, - }, - } - if err := p.db.InsertFork(pe); err != nil { - p.logger.Errorf("insert fork: %w", err) - continue - } - case ebpfevents.EventTypeProcessExec: - body, ok := ev.Body.(*ebpfevents.ProcessExec) - if !ok { - logger.Errorf("unexpected event body") - continue - } - pe := types.ProcessExecEvent{ - Pids: types.PidInfo{ - Tid: body.Pids.Tid, - Tgid: body.Pids.Tgid, - Ppid: body.Pids.Ppid, - Pgid: body.Pids.Pgid, - Sid: body.Pids.Sid, - StartTimeNs: body.Pids.StartTimeNs, - }, - Creds: types.CredInfo{ - Ruid: body.Creds.Ruid, - Rgid: body.Creds.Rgid, - Euid: body.Creds.Euid, - Egid: body.Creds.Egid, - Suid: body.Creds.Suid, - Sgid: body.Creds.Sgid, - CapPermitted: body.Creds.CapPermitted, - CapEffective: body.Creds.CapEffective, - }, - CTty: types.TtyDev{ - Major: body.CTTY.Major, - Minor: body.CTTY.Minor, - }, - Cwd: body.Cwd, - Argv: deepcopy.Copy(body.Argv).([]string), - Env: deepcopy.Copy(body.Env).(map[string]string), - Filename: body.Filename, - } - if err := p.db.InsertExec(pe); err != nil { - p.logger.Errorf("insert exec: %w", err) - continue - } - case ebpfevents.EventTypeProcessExit: - body, ok := ev.Body.(*ebpfevents.ProcessExit) - if !ok { - logger.Errorf("unexpected event body") - continue - } - pe := types.ProcessExitEvent{ - Pids: types.PidInfo{ - Tid: body.Pids.Tid, - Tgid: body.Pids.Tgid, - Ppid: body.Pids.Ppid, - Pgid: body.Pids.Pgid, - Sid: body.Pids.Sid, - StartTimeNs: body.Pids.StartTimeNs, - }, - ExitCode: body.ExitCode, - } - if err := p.db.InsertExit(pe); err != nil { - p.logger.Errorf("insert exit: %w", err) - continue - } + pe := types.ProcessForkEvent{ + ParentPids: types.PidInfo{ + Tid: body.ParentPids.Tid, + Tgid: body.ParentPids.Tgid, + Ppid: body.ParentPids.Ppid, + Pgid: body.ParentPids.Pgid, + Sid: body.ParentPids.Sid, + StartTimeNs: body.ParentPids.StartTimeNs, + }, + ChildPids: types.PidInfo{ + Tid: body.ChildPids.Tid, + Tgid: body.ChildPids.Tgid, + Ppid: body.ChildPids.Ppid, + Pgid: body.ChildPids.Pgid, + Sid: body.ChildPids.Sid, + StartTimeNs: body.ChildPids.StartTimeNs, + }, + Creds: types.CredInfo{ + Ruid: body.Creds.Ruid, + Rgid: body.Creds.Rgid, + Euid: body.Creds.Euid, + Egid: body.Creds.Egid, + Suid: body.Creds.Suid, + Sgid: body.Creds.Sgid, + CapPermitted: body.Creds.CapPermitted, + CapEffective: body.Creds.CapEffective, + }, + } + if err := p.db.InsertFork(pe); err != nil { + p.logger.Errorf("insert fork: %w", err) + continue + } + case ebpfevents.EventTypeProcessExec: + body, ok := ev.Body.(*ebpfevents.ProcessExec) + if !ok { + logger.Errorf("unexpected event body") + continue + } + pe := types.ProcessExecEvent{ + Pids: types.PidInfo{ + Tid: body.Pids.Tid, + Tgid: body.Pids.Tgid, + Ppid: body.Pids.Ppid, + Pgid: body.Pids.Pgid, + Sid: body.Pids.Sid, + StartTimeNs: body.Pids.StartTimeNs, + }, + Creds: types.CredInfo{ + Ruid: body.Creds.Ruid, + Rgid: body.Creds.Rgid, + Euid: body.Creds.Euid, + Egid: body.Creds.Egid, + Suid: body.Creds.Suid, + Sgid: body.Creds.Sgid, + CapPermitted: body.Creds.CapPermitted, + CapEffective: body.Creds.CapEffective, + }, + CTty: types.TtyDev{ + Major: body.CTTY.Major, + Minor: body.CTTY.Minor, + }, + Cwd: body.Cwd, + Argv: deepcopy.Copy(body.Argv).([]string), + Env: deepcopy.Copy(body.Env).(map[string]string), + Filename: body.Filename, + } + if err := p.db.InsertExec(pe); err != nil { + p.logger.Errorf("insert exec: %w", err) + continue + } + case ebpfevents.EventTypeProcessExit: + body, ok := ev.Body.(*ebpfevents.ProcessExit) + if !ok { + logger.Errorf("unexpected event body") + continue + } + pe := types.ProcessExitEvent{ + Pids: types.PidInfo{ + Tid: body.Pids.Tid, + Tgid: body.Pids.Tgid, + Ppid: body.Pids.Ppid, + Pgid: body.Pids.Pgid, + Sid: body.Pids.Sid, + StartTimeNs: body.Pids.StartTimeNs, + }, + ExitCode: body.ExitCode, + } + if err := p.db.InsertExit(pe); err != nil { + p.logger.Errorf("insert exit: %w", err) + continue } - continue } } }(*p.logger) From e9aea4dc292c55988f585fb908bce1514aa3852c Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Wed, 24 Jan 2024 17:10:50 -0800 Subject: [PATCH 18/32] Use watcher for ebpf events Use watcher, which provides singleton access for ebpfevents --- .../auditbeat/internal/ebpf/seccomp_linux.go | 27 +++ x-pack/auditbeat/internal/ebpf/types.go | 16 ++ .../auditbeat/internal/ebpf/watcher_linux.go | 154 ++++++++++++++++++ .../auditbeat/internal/ebpf/watcher_other.go | 15 ++ .../auditbeat/internal/ebpf/watcher_test.go | 48 ++++++ .../add_session_metadata.go | 8 +- ...empty.go => add_session_metadata_other.go} | 2 + .../add_session_metadata_test.go | 1 - .../add_session_metadata/processdb/db_test.go | 3 +- .../add_session_metadata/procfs/procfs.go | 3 +- .../provider/ebpf_provider/ebpf_provider.go | 19 ++- .../add_session_metadata/timeutils/time.go | 4 +- 12 files changed, 279 insertions(+), 21 deletions(-) create mode 100644 x-pack/auditbeat/internal/ebpf/seccomp_linux.go create mode 100644 x-pack/auditbeat/internal/ebpf/types.go create mode 100644 x-pack/auditbeat/internal/ebpf/watcher_linux.go create mode 100644 x-pack/auditbeat/internal/ebpf/watcher_other.go create mode 100644 x-pack/auditbeat/internal/ebpf/watcher_test.go rename x-pack/auditbeat/processors/add_session_metadata/{empty.go => add_session_metadata_other.go} (93%) diff --git a/x-pack/auditbeat/internal/ebpf/seccomp_linux.go b/x-pack/auditbeat/internal/ebpf/seccomp_linux.go new file mode 100644 index 000000000000..8adbd8ad8dfc --- /dev/null +++ b/x-pack/auditbeat/internal/ebpf/seccomp_linux.go @@ -0,0 +1,27 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build linux + +package ebpf + +import ( + "runtime" + + "github.com/elastic/beats/v7/libbeat/common/seccomp" +) + +func init() { + switch runtime.GOARCH { + case "amd64", "arm64": + syscalls := []string{ + "bpf", + "eventfd2", // needed by ringbuf + "perf_event_open", // needed by tracepoints + } + if err := seccomp.ModifyDefaultPolicy(seccomp.AddSyscall, syscalls...); err != nil { + panic(err) + } + } +} diff --git a/x-pack/auditbeat/internal/ebpf/types.go b/x-pack/auditbeat/internal/ebpf/types.go new file mode 100644 index 000000000000..8dec9a8401d2 --- /dev/null +++ b/x-pack/auditbeat/internal/ebpf/types.go @@ -0,0 +1,16 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package ebpf + +import ( + "github.com/elastic/ebpfevents" +) + +type EventMask uint64 + +type Watcher interface { + Subscribe(string, EventMask) <-chan ebpfevents.Record + Unsubscribe(string) +} diff --git a/x-pack/auditbeat/internal/ebpf/watcher_linux.go b/x-pack/auditbeat/internal/ebpf/watcher_linux.go new file mode 100644 index 000000000000..2c5d91cd9d75 --- /dev/null +++ b/x-pack/auditbeat/internal/ebpf/watcher_linux.go @@ -0,0 +1,154 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build linux + +package ebpf + +import ( + "context" + "fmt" + "sync" + + "github.com/elastic/ebpfevents" +) + +var ( + gWatcherOnce sync.Once + gWatcherErr error + gWatcher watcher +) + +type client struct { + name string + mask EventMask + records chan ebpfevents.Record +} + +type watcher struct { + sync.Mutex + ctx context.Context + cancel context.CancelFunc + loader *ebpfevents.Loader + clients map[string]client + status status +} + +type status int + +const ( + stopped status = iota + started +) + +func GetWatcher() (Watcher, error) { + gWatcher.Lock() + defer gWatcher.Unlock() + + // Try to load the probe once on startup so consumers can error out. + gWatcherOnce.Do(func() { + if gWatcher.status == stopped { + l, err := ebpfevents.NewLoader() + if err != nil { + gWatcherErr = fmt.Errorf("init ebpf loader: %w", err) + return + } + _ = l.Close() + } + }) + + return &gWatcher, gWatcherErr +} + +func (w *watcher) Subscribe(name string, events EventMask) <-chan ebpfevents.Record { + w.Lock() + defer w.Unlock() + + if w.status == stopped { + startLocked() + } + + w.clients[name] = client{ + name: name, + mask: events, + records: make(chan ebpfevents.Record), + } + + return w.clients[name].records +} + +func (w *watcher) Unsubscribe(name string) { + w.Lock() + defer w.Unlock() + + delete(w.clients, name) + + if w.nclients() == 0 { + stopLocked() + } +} + +func startLocked() { + loader, err := ebpfevents.NewLoader() + if err != nil { + gWatcherErr = fmt.Errorf("start ebpf loader: %w", err) + return + } + + gWatcher.loader = loader + gWatcher.clients = make(map[string]client) + + records := make(chan ebpfevents.Record) + gWatcher.ctx, gWatcher.cancel = context.WithCancel(context.Background()) + + go gWatcher.loader.EventLoop(gWatcher.ctx, records) + go func() { + for { + select { + case record := <-records: + if record.Error != nil { + for _, client := range gWatcher.clients { + client.records <- record + } + continue + } + for _, client := range gWatcher.clients { + if client.mask&EventMask(record.Event.Type) != 0 { + client.records <- record + } + } + continue + case <-gWatcher.ctx.Done(): + return + } + } + }() + + gWatcher.status = started +} + +func stopLocked() { + _ = gWatcher.close() + gWatcher.status = stopped +} + +func (w *watcher) nclients() int { + return len(w.clients) +} + +func (w *watcher) close() error { + if w.cancel != nil { + w.cancel() + } + + if w.loader != nil { + _ = w.loader.Close() + } + + for _, cl := range w.clients { + close(cl.records) + } + + return nil +} diff --git a/x-pack/auditbeat/internal/ebpf/watcher_other.go b/x-pack/auditbeat/internal/ebpf/watcher_other.go new file mode 100644 index 000000000000..8d13ba135962 --- /dev/null +++ b/x-pack/auditbeat/internal/ebpf/watcher_other.go @@ -0,0 +1,15 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build !linux + +package ebpf + +import "errors" + +var ErrNotSupported = errors.New("not supported") + +func NewWatcher() (Watcher, error) { + return nil, ErrNotSupported +} diff --git a/x-pack/auditbeat/internal/ebpf/watcher_test.go b/x-pack/auditbeat/internal/ebpf/watcher_test.go new file mode 100644 index 000000000000..0fa13cebcef1 --- /dev/null +++ b/x-pack/auditbeat/internal/ebpf/watcher_test.go @@ -0,0 +1,48 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build linux + +package ebpf + +import ( + "math" + "testing" + + "github.com/stretchr/testify/assert" +) + +const allEvents = EventMask(math.MaxUint64) + +func TestWatcherStartStop(t *testing.T) { + w, err := GetWatcher() + if err != nil { + t.Skipf("skipping ebpf watcher test: %v", err) + } + assert.Equal(t, gWatcher.status, stopped) + assert.Equal(t, 0, gWatcher.nclients()) + + _ = w.Subscribe("test-1", allEvents) + assert.Equal(t, gWatcher.status, started) + assert.Equal(t, 1, gWatcher.nclients()) + + _ = w.Subscribe("test-2", allEvents) + assert.Equal(t, 2, gWatcher.nclients()) + + w.Unsubscribe("test-2") + assert.Equal(t, 1, gWatcher.nclients()) + + w.Unsubscribe("dummy") + assert.Equal(t, 1, gWatcher.nclients()) + + assert.Equal(t, gWatcher.status, started) + w.Unsubscribe("test-1") + assert.Equal(t, 0, gWatcher.nclients()) + assert.Equal(t, gWatcher.status, stopped) + + _ = w.Subscribe("new", allEvents) + assert.Equal(t, 1, gWatcher.nclients()) + assert.Equal(t, gWatcher.status, started) + w.Unsubscribe("new") +} diff --git a/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go b/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go index 05b97fca295a..5db5b2388fc3 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go +++ b/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go @@ -13,17 +13,15 @@ import ( "strconv" "time" - "github.com/elastic/elastic-agent-libs/mapstr" - + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/processors" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/processdb" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/procfs" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/provider" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/provider/ebpf_provider" - - "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/beats/v7/libbeat/processors" "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/mapstr" ) const ( diff --git a/x-pack/auditbeat/processors/add_session_metadata/empty.go b/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata_other.go similarity index 93% rename from x-pack/auditbeat/processors/add_session_metadata/empty.go rename to x-pack/auditbeat/processors/add_session_metadata/add_session_metadata_other.go index 2454fb84c137..5269e7c707c8 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/empty.go +++ b/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata_other.go @@ -2,4 +2,6 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +//go:build !linux + package add_session_metadata diff --git a/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata_test.go b/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata_test.go index 8135ce9f9f6c..dfcf553c2da4 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata_test.go +++ b/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata_test.go @@ -16,7 +16,6 @@ import ( "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/processdb" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/procfs" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/types" - "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" ) diff --git a/x-pack/auditbeat/processors/add_session_metadata/processdb/db_test.go b/x-pack/auditbeat/processors/add_session_metadata/processdb/db_test.go index a6c041710fc6..9e283d45e9ee 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/processdb/db_test.go +++ b/x-pack/auditbeat/processors/add_session_metadata/processdb/db_test.go @@ -10,10 +10,9 @@ import ( "testing" "github.com/stretchr/testify/assert" + "golang.org/x/sys/unix" "github.com/elastic/elastic-agent-libs/logp" - - "golang.org/x/sys/unix" ) var logger = logp.NewLogger("processdb") diff --git a/x-pack/auditbeat/processors/add_session_metadata/procfs/procfs.go b/x-pack/auditbeat/processors/add_session_metadata/procfs/procfs.go index 647986c23bfe..a22c7320ff69 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/procfs/procfs.go +++ b/x-pack/auditbeat/processors/add_session_metadata/procfs/procfs.go @@ -12,12 +12,11 @@ import ( "strings" "github.com/prometheus/procfs" + "golang.org/x/sys/unix" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/timeutils" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/types" "github.com/elastic/elastic-agent-libs/logp" - - "golang.org/x/sys/unix" ) func MajorTty(ttyNr uint32) uint16 { diff --git a/x-pack/auditbeat/processors/add_session_metadata/provider/ebpf_provider/ebpf_provider.go b/x-pack/auditbeat/processors/add_session_metadata/provider/ebpf_provider/ebpf_provider.go index 69f28df20f35..c8bafec927e5 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/provider/ebpf_provider/ebpf_provider.go +++ b/x-pack/auditbeat/processors/add_session_metadata/provider/ebpf_provider/ebpf_provider.go @@ -8,18 +8,22 @@ package ebpf_provider import ( "context" + "fmt" "github.com/mohae/deepcopy" "github.com/elastic/beats/v7/libbeat/beat" - + "github.com/elastic/beats/v7/x-pack/auditbeat/internal/ebpf" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/processdb" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/provider" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/types" - + "github.com/elastic/ebpfevents" "github.com/elastic/elastic-agent-libs/logp" +) - "github.com/elastic/ebpfevents" +const ( + name = "add_session_metadata" + allEvents = ebpf.EventMask(ebpfevents.EventTypeProcessFork & ebpfevents.EventTypeProcessExec & ebpfevents.EventTypeProcessExit) ) type prvdr struct { @@ -35,15 +39,12 @@ func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (pr db: db, } - l, err := ebpfevents.NewLoader() + w, err := ebpf.GetWatcher() if err != nil { - p.logger.Errorf("loading ebpf: %w", err) - return prvdr{}, err + return nil, fmt.Errorf("get ebpf watcher: %w", err) } - records := make(chan ebpfevents.Record) - - go l.EventLoop(p.ctx, records) + records := w.Subscribe(name, allEvents) go func(logger logp.Logger) { for { diff --git a/x-pack/auditbeat/processors/add_session_metadata/timeutils/time.go b/x-pack/auditbeat/processors/add_session_metadata/timeutils/time.go index 257d49f4dc53..5328717b99b6 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/timeutils/time.go +++ b/x-pack/auditbeat/processors/add_session_metadata/timeutils/time.go @@ -10,7 +10,7 @@ import ( "fmt" "time" - p "github.com/prometheus/procfs" + "github.com/prometheus/procfs" "github.com/tklauser/go-sysconf" ) @@ -20,7 +20,7 @@ var ( ) func mustGetBootTime() time.Time { - fs, err := p.NewDefaultFS() + fs, err := procfs.NewDefaultFS() if err != nil { panic(fmt.Sprintf("could not get procfs: %v", err)) } From c001219b2d89d99a6d92d3564c6c70ef254f0862 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Thu, 25 Jan 2024 16:49:21 -0800 Subject: [PATCH 19/32] remove seccomp init --- .../auditbeat/internal/ebpf/seccomp_linux.go | 27 ------------------- .../provider/ebpf_provider/ebpf_provider.go | 4 +-- 2 files changed, 2 insertions(+), 29 deletions(-) delete mode 100644 x-pack/auditbeat/internal/ebpf/seccomp_linux.go diff --git a/x-pack/auditbeat/internal/ebpf/seccomp_linux.go b/x-pack/auditbeat/internal/ebpf/seccomp_linux.go deleted file mode 100644 index 8adbd8ad8dfc..000000000000 --- a/x-pack/auditbeat/internal/ebpf/seccomp_linux.go +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -//go:build linux - -package ebpf - -import ( - "runtime" - - "github.com/elastic/beats/v7/libbeat/common/seccomp" -) - -func init() { - switch runtime.GOARCH { - case "amd64", "arm64": - syscalls := []string{ - "bpf", - "eventfd2", // needed by ringbuf - "perf_event_open", // needed by tracepoints - } - if err := seccomp.ModifyDefaultPolicy(seccomp.AddSyscall, syscalls...); err != nil { - panic(err) - } - } -} diff --git a/x-pack/auditbeat/processors/add_session_metadata/provider/ebpf_provider/ebpf_provider.go b/x-pack/auditbeat/processors/add_session_metadata/provider/ebpf_provider/ebpf_provider.go index c8bafec927e5..ea40955dcaf7 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/provider/ebpf_provider/ebpf_provider.go +++ b/x-pack/auditbeat/processors/add_session_metadata/provider/ebpf_provider/ebpf_provider.go @@ -23,7 +23,7 @@ import ( const ( name = "add_session_metadata" - allEvents = ebpf.EventMask(ebpfevents.EventTypeProcessFork & ebpfevents.EventTypeProcessExec & ebpfevents.EventTypeProcessExit) + eventMask = ebpf.EventMask(ebpfevents.EventTypeProcessFork | ebpfevents.EventTypeProcessExec | ebpfevents.EventTypeProcessExit) ) type prvdr struct { @@ -44,7 +44,7 @@ func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (pr return nil, fmt.Errorf("get ebpf watcher: %w", err) } - records := w.Subscribe(name, allEvents) + records := w.Subscribe(name, eventMask) go func(logger logp.Logger) { for { From a5986dd20bbae0d57e672d996d731290aa6301e3 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Thu, 25 Jan 2024 16:54:00 -0800 Subject: [PATCH 20/32] Update x-pack/auditbeat/internal/ebpf/watcher_linux.go Co-authored-by: Mattia Meleleo --- x-pack/auditbeat/internal/ebpf/watcher_linux.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/auditbeat/internal/ebpf/watcher_linux.go b/x-pack/auditbeat/internal/ebpf/watcher_linux.go index 2c5d91cd9d75..1c972ce8b145 100644 --- a/x-pack/auditbeat/internal/ebpf/watcher_linux.go +++ b/x-pack/auditbeat/internal/ebpf/watcher_linux.go @@ -99,7 +99,7 @@ func startLocked() { gWatcher.loader = loader gWatcher.clients = make(map[string]client) - records := make(chan ebpfevents.Record) + records := make(chan ebpfevents.Record, loader.BufferLen()) gWatcher.ctx, gWatcher.cancel = context.WithCancel(context.Background()) go gWatcher.loader.EventLoop(gWatcher.ctx, records) From d5da140a88cccd9c1915a1a6d56cab57757eac10 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Sun, 28 Jan 2024 21:34:38 -0800 Subject: [PATCH 21/32] Add seccomp policy modification --- .../auditbeat/internal/ebpf/seccomp_linux.go | 29 +++++++++++++++++++ .../add_session_metadata.go | 7 +++-- .../processors/add_session_metadata/config.go | 2 +- 3 files changed, 35 insertions(+), 3 deletions(-) create mode 100644 x-pack/auditbeat/internal/ebpf/seccomp_linux.go diff --git a/x-pack/auditbeat/internal/ebpf/seccomp_linux.go b/x-pack/auditbeat/internal/ebpf/seccomp_linux.go new file mode 100644 index 000000000000..4771e388afd9 --- /dev/null +++ b/x-pack/auditbeat/internal/ebpf/seccomp_linux.go @@ -0,0 +1,29 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build linux + +package ebpf + +import ( + "runtime" + + "github.com/elastic/beats/v7/libbeat/common/seccomp" +) + +func init() { + switch runtime.GOARCH { + case "amd64", "arm64": + syscalls := []string{ + "bpf", + "eventfd2", // needed by ringbuf + "perf_event_open", // needed by tracepoints + "openat", // needed to create map + "newfstatat", // needed for BTF + } + if err := seccomp.ModifyDefaultPolicy(seccomp.AddSyscall, syscalls...); err != nil { + panic(err) + } + } +} diff --git a/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go b/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go index 5db5b2388fc3..d63894da52fd 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go +++ b/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go @@ -56,6 +56,9 @@ func New(cfg *config.C) (beat.Processor, error) { logger.Debugf("backfilled %d processes", len(backfilledPIDs)) switch c.Backend { + case "auto": + // "auto" always uses ebpf, as it's currently the only backend + fallthrough case "ebpf": p, err := ebpf_provider.NewProvider(ctx, logger, db) if err != nil { @@ -92,7 +95,7 @@ func (p *addSessionMetadata) Run(ev *beat.Event) (*beat.Event, error) { } func (p *addSessionMetadata) String() string { - return fmt.Sprintf("%v=[backend=%s, pid_field=%s, override_fields=%t]", + return fmt.Sprintf("%v=[backend=%s, pid_field=%s, replace_fields=%t]", processorName, p.config.Backend, p.config.PidField, p.config.ReplaceFields) } @@ -173,7 +176,7 @@ func (p *addSessionMetadata) replaceFields(ev *beat.Event) error { // process start syscall, err := ev.Fields.GetValue("auditd.data.syscall") if err != nil { - return err + return nil //nolint:nilerr // processor can be called on unsupported events; not an error } switch syscall { case "execveat": diff --git a/x-pack/auditbeat/processors/add_session_metadata/config.go b/x-pack/auditbeat/processors/add_session_metadata/config.go index f2134edb57d8..08a8276946cf 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/config.go +++ b/x-pack/auditbeat/processors/add_session_metadata/config.go @@ -15,7 +15,7 @@ type Config struct { func defaultConfig() Config { return Config{ - Backend: "ebpf", + Backend: "auto", ReplaceFields: false, PidField: "process.pid", } From 7fe0ba4aad4b7c2a66695dda014151eee0df5009 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Mon, 29 Jan 2024 13:32:47 -0800 Subject: [PATCH 22/32] Use buffered channel in watcher client --- NOTICE.txt | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- x-pack/auditbeat/internal/ebpf/watcher_linux.go | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index dd27366f4fd9..9594ffd92d13 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -12257,11 +12257,11 @@ SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/elastic/ebpfevents -Version: v0.3.0 +Version: v0.3.1 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/ebpfevents@v0.3.0/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/elastic/ebpfevents@v0.3.1/LICENSE.txt: The https://github.com/elastic/ebpfevents repository contains source code under various licenses: diff --git a/go.mod b/go.mod index f804b8a01ad8..9c99c0ec736e 100644 --- a/go.mod +++ b/go.mod @@ -200,7 +200,7 @@ require ( github.com/aws/smithy-go v1.13.5 github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5 github.com/elastic/bayeux v1.0.5 - github.com/elastic/ebpfevents v0.3.0 + github.com/elastic/ebpfevents v0.3.1 github.com/elastic/elastic-agent-autodiscover v0.6.7 github.com/elastic/elastic-agent-libs v0.7.5 github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 diff --git a/go.sum b/go.sum index 828d5d119381..9651b9b0e5e3 100644 --- a/go.sum +++ b/go.sum @@ -660,8 +660,8 @@ github.com/elastic/bayeux v1.0.5 h1:UceFq01ipmT3S8DzFK+uVAkbCdiPR0Bqei8qIGmUeY0= github.com/elastic/bayeux v1.0.5/go.mod h1:CSI4iP7qeo5MMlkznGvYKftp8M7qqP/3nzmVZoXHY68= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY= -github.com/elastic/ebpfevents v0.3.0 h1:tN2X+FNyV2o1x81tX1anAyVwXOTd1iRNxlkhf6zVY24= -github.com/elastic/ebpfevents v0.3.0/go.mod h1:o21z5xup/9dK8u0Hg9bZRflSqqj1Zu5h2dg2hSTcUPQ= +github.com/elastic/ebpfevents v0.3.1 h1:cUP3QXx6MhRGVXWZSgNalY8y5Vd1dgC56DMfeejnXFU= +github.com/elastic/ebpfevents v0.3.1/go.mod h1:o21z5xup/9dK8u0Hg9bZRflSqqj1Zu5h2dg2hSTcUPQ= github.com/elastic/elastic-agent-autodiscover v0.6.7 h1:+KVjltN0rPsBrU8b156gV4lOTBgG/vt0efFCFARrf3g= github.com/elastic/elastic-agent-autodiscover v0.6.7/go.mod h1:hFeFqneS2r4jD0/QzGkrNk0YVdN0JGh7lCWdsH7zcI4= github.com/elastic/elastic-agent-client/v7 v7.6.0 h1:FEn6FjzynW4TIQo5G096Tr7xYK/P5LY9cSS6wRbXZTc= diff --git a/x-pack/auditbeat/internal/ebpf/watcher_linux.go b/x-pack/auditbeat/internal/ebpf/watcher_linux.go index 1c972ce8b145..f751abe8019a 100644 --- a/x-pack/auditbeat/internal/ebpf/watcher_linux.go +++ b/x-pack/auditbeat/internal/ebpf/watcher_linux.go @@ -72,7 +72,7 @@ func (w *watcher) Subscribe(name string, events EventMask) <-chan ebpfevents.Rec w.clients[name] = client{ name: name, mask: events, - records: make(chan ebpfevents.Record), + records: make(chan ebpfevents.Record, w.loader.BufferLen()), } return w.clients[name].records From 9c59a7bb5150acfc630f271492f2061066a994c4 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Mon, 29 Jan 2024 16:03:19 -0800 Subject: [PATCH 23/32] Move ebpf watcher to libbeat --- NOTICE.txt | 4 +- go.mod | 2 +- go.sum | 4 +- libbeat/ebpf/seccomp_linux.go | 42 +++++++++++++++++++ libbeat/ebpf/types.go | 29 +++++++++++++ .../ebpf/watcher_linux.go | 19 +++++++-- libbeat/ebpf/watcher_other.go | 28 +++++++++++++ .../internal => libbeat}/ebpf/watcher_test.go | 19 +++++++-- .../auditbeat/internal/ebpf/seccomp_linux.go | 29 ------------- x-pack/auditbeat/internal/ebpf/types.go | 16 ------- .../auditbeat/internal/ebpf/watcher_other.go | 15 ------- .../provider/ebpf_provider/ebpf_provider.go | 2 +- 12 files changed, 137 insertions(+), 72 deletions(-) create mode 100644 libbeat/ebpf/seccomp_linux.go create mode 100644 libbeat/ebpf/types.go rename {x-pack/auditbeat/internal => libbeat}/ebpf/watcher_linux.go (76%) create mode 100644 libbeat/ebpf/watcher_other.go rename {x-pack/auditbeat/internal => libbeat}/ebpf/watcher_test.go (55%) delete mode 100644 x-pack/auditbeat/internal/ebpf/seccomp_linux.go delete mode 100644 x-pack/auditbeat/internal/ebpf/types.go delete mode 100644 x-pack/auditbeat/internal/ebpf/watcher_other.go diff --git a/NOTICE.txt b/NOTICE.txt index 9594ffd92d13..fadf4cc224bf 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -12257,11 +12257,11 @@ SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/elastic/ebpfevents -Version: v0.3.1 +Version: v0.3.2 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/ebpfevents@v0.3.1/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/elastic/ebpfevents@v0.3.2/LICENSE.txt: The https://github.com/elastic/ebpfevents repository contains source code under various licenses: diff --git a/go.mod b/go.mod index 9c99c0ec736e..ceb49abc9fa1 100644 --- a/go.mod +++ b/go.mod @@ -200,7 +200,7 @@ require ( github.com/aws/smithy-go v1.13.5 github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5 github.com/elastic/bayeux v1.0.5 - github.com/elastic/ebpfevents v0.3.1 + github.com/elastic/ebpfevents v0.3.2 github.com/elastic/elastic-agent-autodiscover v0.6.7 github.com/elastic/elastic-agent-libs v0.7.5 github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 diff --git a/go.sum b/go.sum index 9651b9b0e5e3..e3e4c7f568fb 100644 --- a/go.sum +++ b/go.sum @@ -660,8 +660,8 @@ github.com/elastic/bayeux v1.0.5 h1:UceFq01ipmT3S8DzFK+uVAkbCdiPR0Bqei8qIGmUeY0= github.com/elastic/bayeux v1.0.5/go.mod h1:CSI4iP7qeo5MMlkznGvYKftp8M7qqP/3nzmVZoXHY68= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY= -github.com/elastic/ebpfevents v0.3.1 h1:cUP3QXx6MhRGVXWZSgNalY8y5Vd1dgC56DMfeejnXFU= -github.com/elastic/ebpfevents v0.3.1/go.mod h1:o21z5xup/9dK8u0Hg9bZRflSqqj1Zu5h2dg2hSTcUPQ= +github.com/elastic/ebpfevents v0.3.2 h1:UJ8kW5jw2TpUR5MEMaZ1O62sK9JQ+5xTlj+YpQC6BXc= +github.com/elastic/ebpfevents v0.3.2/go.mod h1:o21z5xup/9dK8u0Hg9bZRflSqqj1Zu5h2dg2hSTcUPQ= github.com/elastic/elastic-agent-autodiscover v0.6.7 h1:+KVjltN0rPsBrU8b156gV4lOTBgG/vt0efFCFARrf3g= github.com/elastic/elastic-agent-autodiscover v0.6.7/go.mod h1:hFeFqneS2r4jD0/QzGkrNk0YVdN0JGh7lCWdsH7zcI4= github.com/elastic/elastic-agent-client/v7 v7.6.0 h1:FEn6FjzynW4TIQo5G096Tr7xYK/P5LY9cSS6wRbXZTc= diff --git a/libbeat/ebpf/seccomp_linux.go b/libbeat/ebpf/seccomp_linux.go new file mode 100644 index 000000000000..0645395a6d7b --- /dev/null +++ b/libbeat/ebpf/seccomp_linux.go @@ -0,0 +1,42 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package ebpf + +import ( + "runtime" + + "github.com/elastic/beats/v7/libbeat/common/seccomp" +) + +func init() { + switch runtime.GOARCH { + case "amd64", "arm64": + syscalls := []string{ + "bpf", + "eventfd2", // needed by ringbuf + "perf_event_open", // needed by tracepoints + "openat", // needed to create map + "newfstatat", // needed for BTF + } + if err := seccomp.ModifyDefaultPolicy(seccomp.AddSyscall, syscalls...); err != nil { + panic(err) + } + } +} diff --git a/libbeat/ebpf/types.go b/libbeat/ebpf/types.go new file mode 100644 index 000000000000..7292972522f0 --- /dev/null +++ b/libbeat/ebpf/types.go @@ -0,0 +1,29 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package ebpf + +import ( + "github.com/elastic/ebpfevents" +) + +type EventMask uint64 + +type Watcher interface { + Subscribe(string, EventMask) <-chan ebpfevents.Record + Unsubscribe(string) +} diff --git a/x-pack/auditbeat/internal/ebpf/watcher_linux.go b/libbeat/ebpf/watcher_linux.go similarity index 76% rename from x-pack/auditbeat/internal/ebpf/watcher_linux.go rename to libbeat/ebpf/watcher_linux.go index f751abe8019a..edf69cd22171 100644 --- a/x-pack/auditbeat/internal/ebpf/watcher_linux.go +++ b/libbeat/ebpf/watcher_linux.go @@ -1,6 +1,19 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. //go:build linux diff --git a/libbeat/ebpf/watcher_other.go b/libbeat/ebpf/watcher_other.go new file mode 100644 index 000000000000..fc9da1b4cb85 --- /dev/null +++ b/libbeat/ebpf/watcher_other.go @@ -0,0 +1,28 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build !linux + +package ebpf + +import "errors" + +var ErrNotSupported = errors.New("not supported") + +func NewWatcher() (Watcher, error) { + return nil, ErrNotSupported +} diff --git a/x-pack/auditbeat/internal/ebpf/watcher_test.go b/libbeat/ebpf/watcher_test.go similarity index 55% rename from x-pack/auditbeat/internal/ebpf/watcher_test.go rename to libbeat/ebpf/watcher_test.go index 0fa13cebcef1..13d27ffd52c0 100644 --- a/x-pack/auditbeat/internal/ebpf/watcher_test.go +++ b/libbeat/ebpf/watcher_test.go @@ -1,6 +1,19 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. //go:build linux diff --git a/x-pack/auditbeat/internal/ebpf/seccomp_linux.go b/x-pack/auditbeat/internal/ebpf/seccomp_linux.go deleted file mode 100644 index 4771e388afd9..000000000000 --- a/x-pack/auditbeat/internal/ebpf/seccomp_linux.go +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -//go:build linux - -package ebpf - -import ( - "runtime" - - "github.com/elastic/beats/v7/libbeat/common/seccomp" -) - -func init() { - switch runtime.GOARCH { - case "amd64", "arm64": - syscalls := []string{ - "bpf", - "eventfd2", // needed by ringbuf - "perf_event_open", // needed by tracepoints - "openat", // needed to create map - "newfstatat", // needed for BTF - } - if err := seccomp.ModifyDefaultPolicy(seccomp.AddSyscall, syscalls...); err != nil { - panic(err) - } - } -} diff --git a/x-pack/auditbeat/internal/ebpf/types.go b/x-pack/auditbeat/internal/ebpf/types.go deleted file mode 100644 index 8dec9a8401d2..000000000000 --- a/x-pack/auditbeat/internal/ebpf/types.go +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package ebpf - -import ( - "github.com/elastic/ebpfevents" -) - -type EventMask uint64 - -type Watcher interface { - Subscribe(string, EventMask) <-chan ebpfevents.Record - Unsubscribe(string) -} diff --git a/x-pack/auditbeat/internal/ebpf/watcher_other.go b/x-pack/auditbeat/internal/ebpf/watcher_other.go deleted file mode 100644 index 8d13ba135962..000000000000 --- a/x-pack/auditbeat/internal/ebpf/watcher_other.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -//go:build !linux - -package ebpf - -import "errors" - -var ErrNotSupported = errors.New("not supported") - -func NewWatcher() (Watcher, error) { - return nil, ErrNotSupported -} diff --git a/x-pack/auditbeat/processors/add_session_metadata/provider/ebpf_provider/ebpf_provider.go b/x-pack/auditbeat/processors/add_session_metadata/provider/ebpf_provider/ebpf_provider.go index ea40955dcaf7..1d8aa2579805 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/provider/ebpf_provider/ebpf_provider.go +++ b/x-pack/auditbeat/processors/add_session_metadata/provider/ebpf_provider/ebpf_provider.go @@ -13,7 +13,7 @@ import ( "github.com/mohae/deepcopy" "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/beats/v7/x-pack/auditbeat/internal/ebpf" + "github.com/elastic/beats/v7/libbeat/ebpf" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/processdb" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/provider" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/types" From 4a304df430b19582c3aca3b9e40a3ff03b9a008e Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Thu, 1 Feb 2024 15:06:00 -0800 Subject: [PATCH 24/32] Remove error return value that was never used --- .../add_session_metadata.go | 2 +- .../add_session_metadata_test.go | 3 +- .../add_session_metadata/processdb/db.go | 97 +++++++++---------- .../processdb/entry_leader_test.go | 21 ++-- .../provider/ebpf_provider/ebpf_provider.go | 15 +-- 5 files changed, 57 insertions(+), 81 deletions(-) diff --git a/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go b/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go index d63894da52fd..e3b58cb54a07 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go +++ b/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go @@ -48,7 +48,7 @@ func New(cfg *config.C) (beat.Processor, error) { logger := logp.NewLogger(logName) - ctx := context.TODO() + ctx := context.Background() reader := procfs.NewProcfsReader(*logger) db := processdb.NewDB(reader, *logger) diff --git a/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata_test.go b/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata_test.go index dfcf553c2da4..c6dc1a31703a 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata_test.go +++ b/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata_test.go @@ -231,8 +231,7 @@ func TestEnrich(t *testing.T) { db := processdb.NewDB(reader, *logger) for _, ev := range tt.mockProcesses { - err := db.InsertExec(ev) - assert.NoError(t, err, "%s: inserting exec to db: %w", tt.testName, err) + db.InsertExec(ev) } s := addSessionMetadata{ logger: logger, diff --git a/x-pack/auditbeat/processors/add_session_metadata/processdb/db.go b/x-pack/auditbeat/processors/add_session_metadata/processdb/db.go index 818e01be8208..99214705568e 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/processdb/db.go +++ b/x-pack/auditbeat/processors/add_session_metadata/processdb/db.go @@ -89,47 +89,47 @@ var ( bootID = mustReadBootID() pidNsInode = mustReadPidNsInode() capNames = []string{ - "CAP_CHOWN", // 0 - "CAP_DAC_OVERRIDE", // 1 - "CAP_DAC_READ_SEARCH", // 2 - "CAP_FOWNER", // 3 - "CAP_FSETID", // 4 - "CAP_KILL", // 5 - "CAP_SETGID", // 6 - "CAP_SETUID", // 7 - "CAP_SETPCAP", // 8 - "CAP_LINUX_IMMUTABLE", // 9 - "CAP_NET_BIND_SERVICE", // 10 - "CAP_NET_BROADCAST", // 11 - "CAP_NET_ADMIN", // 12 - "CAP_NET_RAW", // 13 - "CAP_IPC_LOCK", // 14 - "CAP_IPC_OWNER", // 15 - "CAP_SYS_MODULE", // 16 - "CAP_SYS_RAWIO", // 17 - "CAP_SYS_CHROOT", // 18 - "CAP_SYS_PTRACE", // 19 - "CAP_SYS_PACCT", // 20 - "CAP_SYS_ADMIN", // 21 - "CAP_SYS_BOOT", // 22 - "CAP_SYS_NICE", // 23 - "CAP_SYS_RESOURCE", // 24 - "CAP_SYS_TIME", // 25 - "CAP_SYS_TTY_CONFIG", // 26 - "CAP_MKNOD", // 27 - "CAP_LEASE", // 28 - "CAP_AUDIT_WRITE", // 29 - "CAP_AUDIT_CONTROL", // 30 - "CAP_SETFCAP", // 31 - "CAP_MAC_OVERRIDE", // 32 - "CAP_MAC_ADMIN", // 33 - "CAP_SYSLOG", // 34 - "CAP_WAKE_ALARM", // 35 - "CAP_BLOCK_SUSPEND", // 36 - "CAP_AUDIT_READ", // 37 - "CAP_PERFMON", // 38 - "CAP_BPF", // 39 - "CAP_CHECKPOINT_RESTORE", // 40 + 0: "CAP_CHOWN", + 1: "CAP_DAC_OVERRIDE", + 2: "CAP_DAC_READ_SEARCH", + 3: "CAP_FOWNER", + 4: "CAP_FSETID", + 5: "CAP_KILL", + 6: "CAP_SETGID", + 7: "CAP_SETUID", + 8: "CAP_SETPCAP", + 9: "CAP_LINUX_IMMUTABLE", + 10: "CAP_NET_BIND_SERVICE", + 11: "CAP_NET_BROADCAST", + 12: "CAP_NET_ADMIN", + 13: "CAP_NET_RAW", + 14: "CAP_IPC_LOCK", + 15: "CAP_IPC_OWNER", + 16: "CAP_SYS_MODULE", + 17: "CAP_SYS_RAWIO", + 18: "CAP_SYS_CHROOT", + 19: "CAP_SYS_PTRACE", + 20: "CAP_SYS_PACCT", + 21: "CAP_SYS_ADMIN", + 22: "CAP_SYS_BOOT", + 23: "CAP_SYS_NICE", + 24: "CAP_SYS_RESOURCE", + 25: "CAP_SYS_TIME", + 26: "CAP_SYS_TTY_CONFIG", + 27: "CAP_MKNOD", + 28: "CAP_LEASE", + 29: "CAP_AUDIT_WRITE", + 30: "CAP_AUDIT_CONTROL", + 31: "CAP_SETFCAP", + 32: "CAP_MAC_OVERRIDE", + 33: "CAP_MAC_ADMIN", + 34: "CAP_SYSLOG", + 35: "CAP_WAKE_ALARM", + 36: "CAP_BLOCK_SUSPEND", + 37: "CAP_AUDIT_READ", + 38: "CAP_PERFMON", + 39: "CAP_BPF", + 40: "CAP_CHECKPOINT_RESTORE", // The ECS spec allows for numerical string representation. // The following capability values are not assigned as of Dec 28, 2023. // If they are added in a future kernel, and this slice has not been @@ -276,7 +276,7 @@ func basename(pathStr string) string { return path.Base(pathStr) } -func (db *DB) InsertFork(fork types.ProcessForkEvent) error { +func (db *DB) InsertFork(fork types.ProcessForkEvent) { db.Lock() defer db.Unlock() @@ -295,8 +295,6 @@ func (db *DB) InsertFork(fork types.ProcessForkEvent) error { Creds: credInfoFromProto(fork.Creds), } } - - return nil } func (db *DB) insertProcess(process Process) { @@ -311,7 +309,7 @@ func (db *DB) insertProcess(process Process) { } } -func (db *DB) InsertExec(exec types.ProcessExecEvent) error { +func (db *DB) InsertExec(exec types.ProcessExecEvent) { db.Lock() defer db.Unlock() @@ -330,8 +328,6 @@ func (db *DB) InsertExec(exec types.ProcessExecEvent) error { if entryLeaderPid != nil { db.entryLeaderRelationships[exec.Pids.Tgid] = *entryLeaderPid } - - return nil } func (db *DB) createEntryLeader(pid uint32, entryType EntryType) { @@ -440,7 +436,7 @@ func (db *DB) evaluateEntryLeader(p Process) *uint32 { return nil } -func (db *DB) InsertSetsid(setsid types.ProcessSetsidEvent) error { +func (db *DB) InsertSetsid(setsid types.ProcessSetsidEvent) { db.Lock() defer db.Unlock() @@ -452,11 +448,9 @@ func (db *DB) InsertSetsid(setsid types.ProcessSetsidEvent) error { Pids: pidInfoFromProto(setsid.Pids), } } - - return nil } -func (db *DB) InsertExit(exit types.ProcessExitEvent) error { +func (db *DB) InsertExit(exit types.ProcessExitEvent) { db.Lock() defer db.Unlock() @@ -464,7 +458,6 @@ func (db *DB) InsertExit(exit types.ProcessExitEvent) error { delete(db.processes, pid) delete(db.entryLeaders, pid) delete(db.entryLeaderRelationships, pid) - return nil } // TODO: is this the correct definition? I looked in endpoint and I swear it looks too simple/generalized diff --git a/x-pack/auditbeat/processors/add_session_metadata/processdb/entry_leader_test.go b/x-pack/auditbeat/processors/add_session_metadata/processdb/entry_leader_test.go index f06e7422b414..9538dd44149a 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/processdb/entry_leader_test.go +++ b/x-pack/auditbeat/processors/add_session_metadata/processdb/entry_leader_test.go @@ -166,12 +166,10 @@ func insertForkAndExec(t *testing.T, db *DB, exec types.ProcessExecEvent) { } if fork.ParentPids.Tgid != 0 { - err = db.InsertFork(fork) - require.Nil(t, err) + db.InsertFork(fork) } - err = db.InsertExec(exec) - require.Nil(t, err) + db.InsertExec(exec) } var systemdPath = "/sbin/systemd" @@ -195,7 +193,7 @@ func TestSingleProcessSessionLeaderEntryTypeTerminal(t *testing.T) { pid := uint32(1234) procPath := "/bin/noproc" - err := db.InsertExec(types.ProcessExecEvent{ + db.InsertExec(types.ProcessExecEvent{ Filename: procPath, Pids: types.PidInfo{ Tgid: pid, @@ -206,7 +204,6 @@ func TestSingleProcessSessionLeaderEntryTypeTerminal(t *testing.T) { Minor: 64, }, }) - require.Nil(t, err) requireProcess(t, db, 1234, procPath) requireEntryLeader(t, db, 1234, 1234, Terminal) @@ -478,7 +475,7 @@ func TestSingleProcessOverwriteOldEntryLeader(t *testing.T) { requireEntryLeader(t, db, bashPid, ssmPid, Init) // skiping setsid event and assuming the pids will be updated in this exec - err := db.InsertExec(types.ProcessExecEvent{ + db.InsertExec(types.ProcessExecEvent{ Filename: bashPath, Pids: types.PidInfo{ Tgid: bashPid, @@ -490,7 +487,6 @@ func TestSingleProcessOverwriteOldEntryLeader(t *testing.T) { Minor: 62, }, }) - require.Nil(t, err) requireProcess(t, db, bashPid, bashPath) requireParent(t, db, bashPid, ssmPid) @@ -1156,7 +1152,7 @@ func TestGrepInIsolation(t *testing.T) { grepPid := uint32(1001) - err := db.InsertExec(types.ProcessExecEvent{ + db.InsertExec(types.ProcessExecEvent{ Filename: grepPath, Pids: types.PidInfo{ Tgid: grepPid, @@ -1164,7 +1160,6 @@ func TestGrepInIsolation(t *testing.T) { Sid: grepPid, }, }) - require.Nil(t, err) process, err := db.GetProcess(grepPid) require.Nil(t, err) @@ -1192,7 +1187,7 @@ func TestKernelThreads(t *testing.T) { kthreaddPath := "kthreadd" rcuGpPath := "rcu_gp" - err := db.InsertExec(types.ProcessExecEvent{ + db.InsertExec(types.ProcessExecEvent{ Filename: kthreaddPath, Pids: types.PidInfo{ Tgid: kthreaddPid, @@ -1200,9 +1195,8 @@ func TestKernelThreads(t *testing.T) { Sid: 0, }, }) - require.Nil(t, err) - err = db.InsertExec(types.ProcessExecEvent{ + db.InsertExec(types.ProcessExecEvent{ Filename: rcuGpPath, Pids: types.PidInfo{ Tgid: rcuGpPid, @@ -1210,7 +1204,6 @@ func TestKernelThreads(t *testing.T) { Sid: 0, }, }) - require.Nil(t, err) // kthreadd kthreadd, err := db.GetProcess(kthreaddPid) diff --git a/x-pack/auditbeat/processors/add_session_metadata/provider/ebpf_provider/ebpf_provider.go b/x-pack/auditbeat/processors/add_session_metadata/provider/ebpf_provider/ebpf_provider.go index 1d8aa2579805..8233b451c230 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/provider/ebpf_provider/ebpf_provider.go +++ b/x-pack/auditbeat/processors/add_session_metadata/provider/ebpf_provider/ebpf_provider.go @@ -92,10 +92,7 @@ func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (pr CapEffective: body.Creds.CapEffective, }, } - if err := p.db.InsertFork(pe); err != nil { - p.logger.Errorf("insert fork: %w", err) - continue - } + p.db.InsertFork(pe) case ebpfevents.EventTypeProcessExec: body, ok := ev.Body.(*ebpfevents.ProcessExec) if !ok { @@ -130,10 +127,7 @@ func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (pr Env: deepcopy.Copy(body.Env).(map[string]string), Filename: body.Filename, } - if err := p.db.InsertExec(pe); err != nil { - p.logger.Errorf("insert exec: %w", err) - continue - } + p.db.InsertExec(pe) case ebpfevents.EventTypeProcessExit: body, ok := ev.Body.(*ebpfevents.ProcessExit) if !ok { @@ -151,10 +145,7 @@ func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (pr }, ExitCode: body.ExitCode, } - if err := p.db.InsertExit(pe); err != nil { - p.logger.Errorf("insert exit: %w", err) - continue - } + p.db.InsertExit(pe) } } }(*p.logger) From 5dbe5dd525827cfdc33dc4bb819bd4dc4233856c Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Fri, 2 Feb 2024 12:16:05 -0800 Subject: [PATCH 25/32] Make capNames an array --- .../add_session_metadata/processdb/db.go | 48 +++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/x-pack/auditbeat/processors/add_session_metadata/processdb/db.go b/x-pack/auditbeat/processors/add_session_metadata/processdb/db.go index 99214705568e..ab679f6f9f37 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/processdb/db.go +++ b/x-pack/auditbeat/processors/add_session_metadata/processdb/db.go @@ -88,7 +88,7 @@ var ( // Fail fast on startup if we can't read them. bootID = mustReadBootID() pidNsInode = mustReadPidNsInode() - capNames = []string{ + capNames = [...]string{ 0: "CAP_CHOWN", 1: "CAP_DAC_OVERRIDE", 2: "CAP_DAC_READ_SEARCH", @@ -134,29 +134,29 @@ var ( // The following capability values are not assigned as of Dec 28, 2023. // If they are added in a future kernel, and this slice has not been // updated, the numerical string will used. - "41", - "42", - "43", - "44", - "45", - "46", - "47", - "48", - "49", - "50", - "51", - "52", - "53", - "54", - "55", - "56", - "57", - "58", - "59", - "60", - "61", - "62", - "63", + 41: "41", + 42: "42", + 43: "43", + 44: "44", + 45: "45", + 46: "46", + 47: "47", + 48: "48", + 49: "49", + 50: "50", + 51: "51", + 52: "52", + 53: "53", + 54: "54", + 55: "55", + 56: "56", + 57: "57", + 58: "58", + 59: "59", + 60: "60", + 61: "61", + 62: "62", + 63: "63", } ) From eeab3975daa80906de926df08b7ab8fa24df6a66 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Wed, 7 Feb 2024 11:12:44 -0800 Subject: [PATCH 26/32] Update ebpfevents lib --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 3a9dd6708a49..3113e5312c59 100644 --- a/go.mod +++ b/go.mod @@ -200,7 +200,7 @@ require ( github.com/aws/smithy-go v1.13.5 github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5 github.com/elastic/bayeux v1.0.5 - github.com/elastic/ebpfevents v0.3.2 + github.com/elastic/ebpfevents v0.4.0 github.com/elastic/elastic-agent-autodiscover v0.6.7 github.com/elastic/elastic-agent-libs v0.7.5 github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 diff --git a/go.sum b/go.sum index 007e01d1133c..285b20d0137d 100644 --- a/go.sum +++ b/go.sum @@ -661,6 +661,8 @@ github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqr github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY= github.com/elastic/ebpfevents v0.3.2 h1:UJ8kW5jw2TpUR5MEMaZ1O62sK9JQ+5xTlj+YpQC6BXc= github.com/elastic/ebpfevents v0.3.2/go.mod h1:o21z5xup/9dK8u0Hg9bZRflSqqj1Zu5h2dg2hSTcUPQ= +github.com/elastic/ebpfevents v0.4.0 h1:M80eAeJnzvGQgU9cjJqkjFca9pjM3aq/TuZxJeom4bI= +github.com/elastic/ebpfevents v0.4.0/go.mod h1:o21z5xup/9dK8u0Hg9bZRflSqqj1Zu5h2dg2hSTcUPQ= github.com/elastic/elastic-agent-autodiscover v0.6.7 h1:+KVjltN0rPsBrU8b156gV4lOTBgG/vt0efFCFARrf3g= github.com/elastic/elastic-agent-autodiscover v0.6.7/go.mod h1:hFeFqneS2r4jD0/QzGkrNk0YVdN0JGh7lCWdsH7zcI4= github.com/elastic/elastic-agent-client/v7 v7.8.0 h1:GHFzDJIWpdgI0qDk5EcqbQJGvwTsl2E2vQK3/xe+MYQ= From 565eaa1fef567a68b1448bb949fbd17d9ff8bc6d Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Tue, 20 Feb 2024 12:09:29 -0800 Subject: [PATCH 27/32] Rename package --- NOTICE.txt | 31 ------------------- auditbeat/cmd/root.go | 3 -- go.mod | 1 - go.sum | 4 --- x-pack/auditbeat/cmd/root.go | 3 ++ .../add_session_metadata.go | 14 ++++----- .../add_session_metadata_test.go | 8 ++--- .../config.go | 2 +- .../doc.go} | 6 ++-- .../processdb/db.go | 6 ++-- .../processdb/db_test.go | 0 .../processdb/entry_leader_test.go | 4 +-- .../procfs/mock.go | 0 .../procfs/procfs.go | 4 +-- .../provider/ebpf_provider/ebpf_provider.go | 16 +++++----- .../provider/provider.go | 0 .../timeutils/time.go | 0 .../timeutils/time_test.go | 0 .../types/events.go | 0 .../types/process.go | 0 20 files changed, 31 insertions(+), 71 deletions(-) rename x-pack/auditbeat/processors/{add_session_metadata => sessionmd}/add_session_metadata.go (92%) rename x-pack/auditbeat/processors/{add_session_metadata => sessionmd}/add_session_metadata_test.go (95%) rename x-pack/auditbeat/processors/{add_session_metadata => sessionmd}/config.go (95%) rename x-pack/auditbeat/processors/{add_session_metadata/add_session_metadata_other.go => sessionmd/doc.go} (60%) rename x-pack/auditbeat/processors/{add_session_metadata => sessionmd}/processdb/db.go (98%) rename x-pack/auditbeat/processors/{add_session_metadata => sessionmd}/processdb/db_test.go (100%) rename x-pack/auditbeat/processors/{add_session_metadata => sessionmd}/processdb/entry_leader_test.go (99%) rename x-pack/auditbeat/processors/{add_session_metadata => sessionmd}/procfs/mock.go (100%) rename x-pack/auditbeat/processors/{add_session_metadata => sessionmd}/procfs/procfs.go (97%) rename x-pack/auditbeat/processors/{add_session_metadata => sessionmd}/provider/ebpf_provider/ebpf_provider.go (88%) rename x-pack/auditbeat/processors/{add_session_metadata => sessionmd}/provider/provider.go (100%) rename x-pack/auditbeat/processors/{add_session_metadata => sessionmd}/timeutils/time.go (100%) rename x-pack/auditbeat/processors/{add_session_metadata => sessionmd}/timeutils/time_test.go (100%) rename x-pack/auditbeat/processors/{add_session_metadata => sessionmd}/types/events.go (100%) rename x-pack/auditbeat/processors/{add_session_metadata => sessionmd}/types/process.go (100%) diff --git a/NOTICE.txt b/NOTICE.txt index c7a089fef75b..a83bc863b52b 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -21420,37 +21420,6 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. --------------------------------------------------------------------------------- -Dependency : github.com/mohae/deepcopy -Version: v0.0.0-20170929034955-c48cc78d4826 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/mohae/deepcopy@v0.0.0-20170929034955-c48cc78d4826/LICENSE: - -The MIT License (MIT) - -Copyright (c) 2014 Joel - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - - -------------------------------------------------------------------------------- Dependency : github.com/olekukonko/tablewriter Version: v0.0.5 diff --git a/auditbeat/cmd/root.go b/auditbeat/cmd/root.go index 4c8c723c4539..0ddc7b8674da 100644 --- a/auditbeat/cmd/root.go +++ b/auditbeat/cmd/root.go @@ -30,9 +30,6 @@ import ( "github.com/elastic/beats/v7/metricbeat/beater" "github.com/elastic/beats/v7/metricbeat/mb/module" "github.com/elastic/elastic-agent-libs/mapstr" - - // Import processors - _ "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata" ) const ( diff --git a/go.mod b/go.mod index af30b15cdf3f..7027313ca1fb 100644 --- a/go.mod +++ b/go.mod @@ -218,7 +218,6 @@ require ( github.com/gorilla/websocket v1.4.2 github.com/icholy/digest v0.1.22 github.com/lestrrat-go/jwx/v2 v2.0.19 - github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 github.com/otiai10/copy v1.12.0 github.com/pierrec/lz4/v4 v4.1.18 github.com/pkg/xattr v0.4.9 diff --git a/go.sum b/go.sum index 5f5fbebb0167..746d5023ae99 100644 --- a/go.sum +++ b/go.sum @@ -663,8 +663,6 @@ github.com/elastic/bayeux v1.0.5 h1:UceFq01ipmT3S8DzFK+uVAkbCdiPR0Bqei8qIGmUeY0= github.com/elastic/bayeux v1.0.5/go.mod h1:CSI4iP7qeo5MMlkznGvYKftp8M7qqP/3nzmVZoXHY68= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY= -github.com/elastic/ebpfevents v0.3.2 h1:UJ8kW5jw2TpUR5MEMaZ1O62sK9JQ+5xTlj+YpQC6BXc= -github.com/elastic/ebpfevents v0.3.2/go.mod h1:o21z5xup/9dK8u0Hg9bZRflSqqj1Zu5h2dg2hSTcUPQ= github.com/elastic/ebpfevents v0.4.0 h1:M80eAeJnzvGQgU9cjJqkjFca9pjM3aq/TuZxJeom4bI= github.com/elastic/ebpfevents v0.4.0/go.mod h1:o21z5xup/9dK8u0Hg9bZRflSqqj1Zu5h2dg2hSTcUPQ= github.com/elastic/elastic-agent-autodiscover v0.6.7 h1:+KVjltN0rPsBrU8b156gV4lOTBgG/vt0efFCFARrf3g= @@ -1501,8 +1499,6 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= -github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= -github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/montanaflynn/stats v0.6.6/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= diff --git a/x-pack/auditbeat/cmd/root.go b/x-pack/auditbeat/cmd/root.go index 60382602060e..4a9b32b56f14 100644 --- a/x-pack/auditbeat/cmd/root.go +++ b/x-pack/auditbeat/cmd/root.go @@ -20,6 +20,9 @@ import ( // Register Auditbeat x-pack modules. _ "github.com/elastic/beats/v7/x-pack/auditbeat/include" _ "github.com/elastic/beats/v7/x-pack/libbeat/include" + + // Import processors + _ "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd" ) // Name of the beat diff --git a/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go b/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go similarity index 92% rename from x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go rename to x-pack/auditbeat/processors/sessionmd/add_session_metadata.go index e3b58cb54a07..2663dd42276f 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata.go +++ b/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go @@ -4,7 +4,7 @@ //go:build linux -package add_session_metadata +package sessionmd import ( "context" @@ -15,10 +15,10 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/processors" - "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/processdb" - "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/procfs" - "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/provider" - "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/provider/ebpf_provider" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/processdb" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/procfs" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider/ebpf_provider" "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" @@ -179,9 +179,7 @@ func (p *addSessionMetadata) replaceFields(ev *beat.Event) error { return nil //nolint:nilerr // processor can be called on unsupported events; not an error } switch syscall { - case "execveat": - fallthrough - case "execve": + case "execveat", "execve": ev.Fields.Put("event.action", []string{"exec", "fork"}) ev.Fields.Put("event.type", []string{"start"}) diff --git a/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata_test.go b/x-pack/auditbeat/processors/sessionmd/add_session_metadata_test.go similarity index 95% rename from x-pack/auditbeat/processors/add_session_metadata/add_session_metadata_test.go rename to x-pack/auditbeat/processors/sessionmd/add_session_metadata_test.go index c6dc1a31703a..d56c390efd40 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata_test.go +++ b/x-pack/auditbeat/processors/sessionmd/add_session_metadata_test.go @@ -4,7 +4,7 @@ //go:build linux -package add_session_metadata +package sessionmd import ( "testing" @@ -13,9 +13,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/processdb" - "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/procfs" - "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/types" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/processdb" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/procfs" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/types" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" ) diff --git a/x-pack/auditbeat/processors/add_session_metadata/config.go b/x-pack/auditbeat/processors/sessionmd/config.go similarity index 95% rename from x-pack/auditbeat/processors/add_session_metadata/config.go rename to x-pack/auditbeat/processors/sessionmd/config.go index 08a8276946cf..845d7821e2d7 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/config.go +++ b/x-pack/auditbeat/processors/sessionmd/config.go @@ -4,7 +4,7 @@ //go:build linux -package add_session_metadata +package sessionmd // Config for add_session_metadata processor. type Config struct { diff --git a/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata_other.go b/x-pack/auditbeat/processors/sessionmd/doc.go similarity index 60% rename from x-pack/auditbeat/processors/add_session_metadata/add_session_metadata_other.go rename to x-pack/auditbeat/processors/sessionmd/doc.go index 5269e7c707c8..6067081c82cb 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/add_session_metadata_other.go +++ b/x-pack/auditbeat/processors/sessionmd/doc.go @@ -2,6 +2,6 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -//go:build !linux - -package add_session_metadata +// sessionmd provides a Beat processor that can enrich process event documents with +// additional session metadata for the processes. +package sessionmd diff --git a/x-pack/auditbeat/processors/add_session_metadata/processdb/db.go b/x-pack/auditbeat/processors/sessionmd/processdb/db.go similarity index 98% rename from x-pack/auditbeat/processors/add_session_metadata/processdb/db.go rename to x-pack/auditbeat/processors/sessionmd/processdb/db.go index ab679f6f9f37..ce67059639b1 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/processdb/db.go +++ b/x-pack/auditbeat/processors/sessionmd/processdb/db.go @@ -20,9 +20,9 @@ import ( "sync" "time" - "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/procfs" - "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/timeutils" - "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/types" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/procfs" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/timeutils" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/types" "github.com/elastic/elastic-agent-libs/logp" ) diff --git a/x-pack/auditbeat/processors/add_session_metadata/processdb/db_test.go b/x-pack/auditbeat/processors/sessionmd/processdb/db_test.go similarity index 100% rename from x-pack/auditbeat/processors/add_session_metadata/processdb/db_test.go rename to x-pack/auditbeat/processors/sessionmd/processdb/db_test.go diff --git a/x-pack/auditbeat/processors/add_session_metadata/processdb/entry_leader_test.go b/x-pack/auditbeat/processors/sessionmd/processdb/entry_leader_test.go similarity index 99% rename from x-pack/auditbeat/processors/add_session_metadata/processdb/entry_leader_test.go rename to x-pack/auditbeat/processors/sessionmd/processdb/entry_leader_test.go index 9538dd44149a..5e28cec8f162 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/processdb/entry_leader_test.go +++ b/x-pack/auditbeat/processors/sessionmd/processdb/entry_leader_test.go @@ -12,8 +12,8 @@ import ( "github.com/stretchr/testify/require" - "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/procfs" - "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/types" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/procfs" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/types" ) const ( diff --git a/x-pack/auditbeat/processors/add_session_metadata/procfs/mock.go b/x-pack/auditbeat/processors/sessionmd/procfs/mock.go similarity index 100% rename from x-pack/auditbeat/processors/add_session_metadata/procfs/mock.go rename to x-pack/auditbeat/processors/sessionmd/procfs/mock.go diff --git a/x-pack/auditbeat/processors/add_session_metadata/procfs/procfs.go b/x-pack/auditbeat/processors/sessionmd/procfs/procfs.go similarity index 97% rename from x-pack/auditbeat/processors/add_session_metadata/procfs/procfs.go rename to x-pack/auditbeat/processors/sessionmd/procfs/procfs.go index a22c7320ff69..29dfc1fe3970 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/procfs/procfs.go +++ b/x-pack/auditbeat/processors/sessionmd/procfs/procfs.go @@ -14,8 +14,8 @@ import ( "github.com/prometheus/procfs" "golang.org/x/sys/unix" - "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/timeutils" - "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/types" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/timeutils" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/types" "github.com/elastic/elastic-agent-libs/logp" ) diff --git a/x-pack/auditbeat/processors/add_session_metadata/provider/ebpf_provider/ebpf_provider.go b/x-pack/auditbeat/processors/sessionmd/provider/ebpf_provider/ebpf_provider.go similarity index 88% rename from x-pack/auditbeat/processors/add_session_metadata/provider/ebpf_provider/ebpf_provider.go rename to x-pack/auditbeat/processors/sessionmd/provider/ebpf_provider/ebpf_provider.go index 8233b451c230..15c180068a53 100644 --- a/x-pack/auditbeat/processors/add_session_metadata/provider/ebpf_provider/ebpf_provider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/ebpf_provider/ebpf_provider.go @@ -10,13 +10,11 @@ import ( "context" "fmt" - "github.com/mohae/deepcopy" - "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/ebpf" - "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/processdb" - "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/provider" - "github.com/elastic/beats/v7/x-pack/auditbeat/processors/add_session_metadata/types" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/processdb" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/types" "github.com/elastic/ebpfevents" "github.com/elastic/elastic-agent-libs/logp" ) @@ -50,7 +48,7 @@ func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (pr for { r := <-records if r.Error != nil { - logger.Errorf("recv'd error: %w", err) + logger.Warnw("received error from the ebpf subscription", "error", err) continue } if r.Event == nil { @@ -61,7 +59,7 @@ func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (pr case ebpfevents.EventTypeProcessFork: body, ok := ev.Body.(*ebpfevents.ProcessFork) if !ok { - logger.Errorf("unexpected event body") + logger.Errorf("unexpected event body, got %T", ev.Body) continue } pe := types.ProcessForkEvent{ @@ -123,8 +121,8 @@ func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (pr Minor: body.CTTY.Minor, }, Cwd: body.Cwd, - Argv: deepcopy.Copy(body.Argv).([]string), - Env: deepcopy.Copy(body.Env).(map[string]string), + Argv: body.Argv, + Env: body.Env, Filename: body.Filename, } p.db.InsertExec(pe) diff --git a/x-pack/auditbeat/processors/add_session_metadata/provider/provider.go b/x-pack/auditbeat/processors/sessionmd/provider/provider.go similarity index 100% rename from x-pack/auditbeat/processors/add_session_metadata/provider/provider.go rename to x-pack/auditbeat/processors/sessionmd/provider/provider.go diff --git a/x-pack/auditbeat/processors/add_session_metadata/timeutils/time.go b/x-pack/auditbeat/processors/sessionmd/timeutils/time.go similarity index 100% rename from x-pack/auditbeat/processors/add_session_metadata/timeutils/time.go rename to x-pack/auditbeat/processors/sessionmd/timeutils/time.go diff --git a/x-pack/auditbeat/processors/add_session_metadata/timeutils/time_test.go b/x-pack/auditbeat/processors/sessionmd/timeutils/time_test.go similarity index 100% rename from x-pack/auditbeat/processors/add_session_metadata/timeutils/time_test.go rename to x-pack/auditbeat/processors/sessionmd/timeutils/time_test.go diff --git a/x-pack/auditbeat/processors/add_session_metadata/types/events.go b/x-pack/auditbeat/processors/sessionmd/types/events.go similarity index 100% rename from x-pack/auditbeat/processors/add_session_metadata/types/events.go rename to x-pack/auditbeat/processors/sessionmd/types/events.go diff --git a/x-pack/auditbeat/processors/add_session_metadata/types/process.go b/x-pack/auditbeat/processors/sessionmd/types/process.go similarity index 100% rename from x-pack/auditbeat/processors/add_session_metadata/types/process.go rename to x-pack/auditbeat/processors/sessionmd/types/process.go From 1f05b145584e0461c3da4b273ce724a30568583e Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Tue, 20 Feb 2024 15:46:39 -0800 Subject: [PATCH 28/32] Use consistant capitialization for initialisms --- .../sessionmd/add_session_metadata.go | 8 +- .../sessionmd/add_session_metadata_test.go | 20 +- .../auditbeat/processors/sessionmd/config.go | 4 +- .../processors/sessionmd/processdb/db.go | 294 +++---- .../processors/sessionmd/processdb/db_test.go | 40 +- .../sessionmd/processdb/entry_leader_test.go | 744 +++++++++--------- .../processors/sessionmd/procfs/procfs.go | 18 +- .../provider/ebpf_provider/ebpf_provider.go | 20 +- .../processors/sessionmd/types/events.go | 30 +- 9 files changed, 532 insertions(+), 646 deletions(-) diff --git a/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go b/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go index 2663dd42276f..231527b88ded 100644 --- a/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go +++ b/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go @@ -76,7 +76,7 @@ func New(cfg *config.C) (beat.Processor, error) { } func (p *addSessionMetadata) Run(ev *beat.Event) (*beat.Event, error) { - _, err := ev.GetValue(p.config.PidField) + _, err := ev.GetValue(p.config.PIDField) if err != nil { // Do not attempt to enrich events without PID; it's not a supported event return ev, nil //nolint:nilerr // Running on events without PID is expected @@ -96,17 +96,17 @@ func (p *addSessionMetadata) Run(ev *beat.Event) (*beat.Event, error) { func (p *addSessionMetadata) String() string { return fmt.Sprintf("%v=[backend=%s, pid_field=%s, replace_fields=%t]", - processorName, p.config.Backend, p.config.PidField, p.config.ReplaceFields) + processorName, p.config.Backend, p.config.PIDField, p.config.ReplaceFields) } func (p *addSessionMetadata) enrich(ev *beat.Event) (*beat.Event, error) { - pidIf, err := ev.GetValue(p.config.PidField) + pidIf, err := ev.GetValue(p.config.PIDField) if err != nil { return nil, err } pid, err := pidToUInt32(pidIf) if err != nil { - return nil, fmt.Errorf("cannot parse pid field '%s': %w", p.config.PidField, err) + return nil, fmt.Errorf("cannot parse pid field '%s': %w", p.config.PIDField, err) } fullProcess, err := p.db.GetProcess(pid) diff --git a/x-pack/auditbeat/processors/sessionmd/add_session_metadata_test.go b/x-pack/auditbeat/processors/sessionmd/add_session_metadata_test.go index d56c390efd40..c8b0e5731565 100644 --- a/x-pack/auditbeat/processors/sessionmd/add_session_metadata_test.go +++ b/x-pack/auditbeat/processors/sessionmd/add_session_metadata_test.go @@ -33,22 +33,22 @@ var ( testName: "Enrich Process", config: Config{ ReplaceFields: false, - PidField: "process.pid", + PIDField: "process.pid", }, mockProcesses: []types.ProcessExecEvent{ { - Pids: types.PidInfo{ + PIDs: types.PIDInfo{ Tid: uint32(100), Tgid: uint32(100), Ppid: uint32(50), Pgid: uint32(100), Sid: uint32(40), }, - Cwd: "/", + CWD: "/", Filename: "/bin/ls", }, { - Pids: types.PidInfo{ + PIDs: types.PIDInfo{ Tid: uint32(50), Tgid: uint32(50), Ppid: uint32(40), @@ -56,7 +56,7 @@ var ( }, }, { - Pids: types.PidInfo{ + PIDs: types.PIDInfo{ Tid: uint32(40), Tgid: uint32(40), Ppid: uint32(1), @@ -92,10 +92,10 @@ var ( expect_error: false, }, { - testName: "No Pid Field in Event", + testName: "No PID Field in Event", config: Config{ ReplaceFields: false, - PidField: "process.pid", + PIDField: "process.pid", }, input: beat.Event{ Fields: mapstr.M{ @@ -111,10 +111,10 @@ var ( expect_error: true, }, { - testName: "Pid Not Number", + testName: "PID Not Number", config: Config{ ReplaceFields: false, - PidField: "process.pid", + PIDField: "process.pid", }, input: beat.Event{ Fields: mapstr.M{ @@ -134,7 +134,7 @@ var ( testName: "PID not in DB", config: Config{ ReplaceFields: false, - PidField: "process.pid", + PIDField: "process.pid", }, input: beat.Event{ Fields: mapstr.M{ diff --git a/x-pack/auditbeat/processors/sessionmd/config.go b/x-pack/auditbeat/processors/sessionmd/config.go index 845d7821e2d7..a082c7d1fd12 100644 --- a/x-pack/auditbeat/processors/sessionmd/config.go +++ b/x-pack/auditbeat/processors/sessionmd/config.go @@ -10,13 +10,13 @@ package sessionmd type Config struct { Backend string `config:"backend"` ReplaceFields bool `config:"replace_fields"` - PidField string `config:"pid_field"` + PIDField string `config:"pid_field"` } func defaultConfig() Config { return Config{ Backend: "auto", ReplaceFields: false, - PidField: "process.pid", + PIDField: "process.pid", } } diff --git a/x-pack/auditbeat/processors/sessionmd/processdb/db.go b/x-pack/auditbeat/processors/sessionmd/processdb/db.go index ce67059639b1..4b0b90ea0046 100644 --- a/x-pack/auditbeat/processors/sessionmd/processdb/db.go +++ b/x-pack/auditbeat/processors/sessionmd/processdb/db.go @@ -10,7 +10,6 @@ import ( "encoding/base64" "errors" "fmt" - "math/bits" "os" "path" "slices" @@ -20,19 +19,20 @@ import ( "sync" "time" + "github.com/elastic/beats/v7/libbeat/common/capabilities" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/procfs" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/timeutils" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/types" "github.com/elastic/elastic-agent-libs/logp" ) -type TtyType int +type TTYType int const ( - TtyUnknown TtyType = iota + TTYUnknown TTYType = iota Pts - Tty - TtyConsole + TTY + TTYConsole ) type EntryType string @@ -74,9 +74,9 @@ const ( ) type Process struct { - Pids types.PidInfo + PIDs types.PIDInfo Creds types.CredInfo - CTty types.TtyDev + CTTY types.TTYDev Argv []string Cwd string Env map[string]string @@ -87,77 +87,7 @@ var ( // The contents of these two files are needed to calculate entity IDs. // Fail fast on startup if we can't read them. bootID = mustReadBootID() - pidNsInode = mustReadPidNsInode() - capNames = [...]string{ - 0: "CAP_CHOWN", - 1: "CAP_DAC_OVERRIDE", - 2: "CAP_DAC_READ_SEARCH", - 3: "CAP_FOWNER", - 4: "CAP_FSETID", - 5: "CAP_KILL", - 6: "CAP_SETGID", - 7: "CAP_SETUID", - 8: "CAP_SETPCAP", - 9: "CAP_LINUX_IMMUTABLE", - 10: "CAP_NET_BIND_SERVICE", - 11: "CAP_NET_BROADCAST", - 12: "CAP_NET_ADMIN", - 13: "CAP_NET_RAW", - 14: "CAP_IPC_LOCK", - 15: "CAP_IPC_OWNER", - 16: "CAP_SYS_MODULE", - 17: "CAP_SYS_RAWIO", - 18: "CAP_SYS_CHROOT", - 19: "CAP_SYS_PTRACE", - 20: "CAP_SYS_PACCT", - 21: "CAP_SYS_ADMIN", - 22: "CAP_SYS_BOOT", - 23: "CAP_SYS_NICE", - 24: "CAP_SYS_RESOURCE", - 25: "CAP_SYS_TIME", - 26: "CAP_SYS_TTY_CONFIG", - 27: "CAP_MKNOD", - 28: "CAP_LEASE", - 29: "CAP_AUDIT_WRITE", - 30: "CAP_AUDIT_CONTROL", - 31: "CAP_SETFCAP", - 32: "CAP_MAC_OVERRIDE", - 33: "CAP_MAC_ADMIN", - 34: "CAP_SYSLOG", - 35: "CAP_WAKE_ALARM", - 36: "CAP_BLOCK_SUSPEND", - 37: "CAP_AUDIT_READ", - 38: "CAP_PERFMON", - 39: "CAP_BPF", - 40: "CAP_CHECKPOINT_RESTORE", - // The ECS spec allows for numerical string representation. - // The following capability values are not assigned as of Dec 28, 2023. - // If they are added in a future kernel, and this slice has not been - // updated, the numerical string will used. - 41: "41", - 42: "42", - 43: "43", - 44: "44", - 45: "45", - 46: "46", - 47: "47", - 48: "48", - 49: "49", - 50: "50", - 51: "51", - 52: "52", - 53: "53", - 54: "54", - 55: "55", - 56: "56", - 57: "57", - 58: "58", - 59: "59", - 60: "60", - 61: "61", - 62: "62", - 63: "63", - } + pidNsInode = mustReadPIDNsInode() ) func mustReadBootID() string { @@ -169,7 +99,7 @@ func mustReadBootID() string { return strings.TrimRight(string(bootID), "\n") } -func mustReadPidNsInode() uint64 { +func mustReadPIDNsInode() uint64 { var ret uint64 pidNsInodeRaw, err := os.Readlink("/proc/self/ns/pid") @@ -184,9 +114,9 @@ func mustReadPidNsInode() uint64 { return ret } -func pidInfoFromProto(p types.PidInfo) types.PidInfo { - return types.PidInfo{ - StartTimeNs: p.StartTimeNs, +func pidInfoFromProto(p types.PIDInfo) types.PIDInfo { + return types.PIDInfo{ + StartTimeNS: p.StartTimeNS, Tid: p.Tid, Tgid: p.Tgid, Vpid: p.Vpid, @@ -209,8 +139,8 @@ func credInfoFromProto(p types.CredInfo) types.CredInfo { } } -func ttyTermiosFromProto(p types.TtyTermios) types.TtyTermios { - return types.TtyTermios{ +func ttyTermiosFromProto(p types.TTYTermios) types.TTYTermios { + return types.TTYTermios{ CIflag: p.CIflag, COflag: p.COflag, CLflag: p.CLflag, @@ -218,15 +148,15 @@ func ttyTermiosFromProto(p types.TtyTermios) types.TtyTermios { } } -func ttyWinsizeFromProto(p types.TtyWinsize) types.TtyWinsize { - return types.TtyWinsize{ +func ttyWinsizeFromProto(p types.TTYWinsize) types.TTYWinsize { + return types.TTYWinsize{ Rows: p.Rows, Cols: p.Cols, } } -func ttyDevFromProto(p types.TtyDev) types.TtyDev { - return types.TtyDev{ +func ttyDevFromProto(p types.TTYDev) types.TTYDev { + return types.TTYDev{ Major: p.Major, Minor: p.Minor, Winsize: ttyWinsizeFromProto(p.Winsize), @@ -280,32 +210,32 @@ func (db *DB) InsertFork(fork types.ProcessForkEvent) { db.Lock() defer db.Unlock() - pid := fork.ChildPids.Tgid - ppid := fork.ParentPids.Tgid + pid := fork.ChildPIDs.Tgid + ppid := fork.ParentPIDs.Tgid if entry, ok := db.processes[ppid]; ok { - entry.Pids = pidInfoFromProto(fork.ChildPids) + entry.PIDs = pidInfoFromProto(fork.ChildPIDs) entry.Creds = credInfoFromProto(fork.Creds) db.processes[pid] = entry - if entryPid, ok := db.entryLeaderRelationships[ppid]; ok { - db.entryLeaderRelationships[pid] = entryPid + if entryPID, ok := db.entryLeaderRelationships[ppid]; ok { + db.entryLeaderRelationships[pid] = entryPID } } else { db.processes[pid] = Process{ - Pids: pidInfoFromProto(fork.ChildPids), + PIDs: pidInfoFromProto(fork.ChildPIDs), Creds: credInfoFromProto(fork.Creds), } } } func (db *DB) insertProcess(process Process) { - pid := process.Pids.Tgid + pid := process.PIDs.Tgid db.processes[pid] = process - entryLeaderPid := db.evaluateEntryLeader(process) - if entryLeaderPid != nil { - db.entryLeaderRelationships[pid] = *entryLeaderPid - db.logger.Debugf("%v name: %s, entry_leader: %d, entry_type: %s", process.Pids, process.Filename, *entryLeaderPid, string(db.entryLeaders[*entryLeaderPid])) + entryLeaderPID := db.evaluateEntryLeader(process) + if entryLeaderPID != nil { + db.entryLeaderRelationships[pid] = *entryLeaderPID + db.logger.Debugf("%v name: %s, entry_leader: %d, entry_type: %s", process.PIDs, process.Filename, *entryLeaderPID, string(db.entryLeaders[*entryLeaderPID])) } else { - db.logger.Debugf("%v name: %s, NO ENTRY LEADER", process.Pids, process.Filename) + db.logger.Debugf("%v name: %s, NO ENTRY LEADER", process.PIDs, process.Filename) } } @@ -314,19 +244,19 @@ func (db *DB) InsertExec(exec types.ProcessExecEvent) { defer db.Unlock() proc := Process{ - Pids: pidInfoFromProto(exec.Pids), + PIDs: pidInfoFromProto(exec.PIDs), Creds: credInfoFromProto(exec.Creds), - CTty: ttyDevFromProto(exec.CTty), + CTTY: ttyDevFromProto(exec.CTTY), Argv: exec.Argv, - Cwd: exec.Cwd, + Cwd: exec.CWD, Env: exec.Env, Filename: exec.Filename, } - db.processes[exec.Pids.Tgid] = proc - entryLeaderPid := db.evaluateEntryLeader(proc) - if entryLeaderPid != nil { - db.entryLeaderRelationships[exec.Pids.Tgid] = *entryLeaderPid + db.processes[exec.PIDs.Tgid] = proc + entryLeaderPID := db.evaluateEntryLeader(proc) + if entryLeaderPID != nil { + db.entryLeaderRelationships[exec.PIDs.Tgid] = *entryLeaderPID } } @@ -337,60 +267,60 @@ func (db *DB) createEntryLeader(pid uint32, entryType EntryType) { // pid returned is a pointer type because its possible for no func (db *DB) evaluateEntryLeader(p Process) *uint32 { - pid := p.Pids.Tgid + pid := p.PIDs.Tgid // init never has an entry leader or meta type - if p.Pids.Tgid == 1 { - db.logger.Debugf("entry_eval %d: process is init, no entry type", p.Pids.Tgid) + if p.PIDs.Tgid == 1 { + db.logger.Debugf("entry_eval %d: process is init, no entry type", p.PIDs.Tgid) return nil } // kernel threads also never have an entry leader or meta type kthreadd // (always pid 2) is the parent of all kernel threads, by filtering pid == // 2 || ppid == 2, we get rid of all of them - if p.Pids.Tgid == 2 || p.Pids.Ppid == 2 { - db.logger.Debugf("entry_eval %d: kernel threads never an entry type (parent is pid 2)", p.Pids.Tgid) + if p.PIDs.Tgid == 2 || p.PIDs.Ppid == 2 { + db.logger.Debugf("entry_eval %d: kernel threads never an entry type (parent is pid 2)", p.PIDs.Tgid) return nil } // could be an entry leader - if p.Pids.Tgid == p.Pids.Sid { - ttyType := getTtyType(p.CTty.Major, p.CTty.Minor) + if p.PIDs.Tgid == p.PIDs.Sid { + ttyType := getTTYType(p.CTTY.Major, p.CTTY.Minor) procBasename := basename(p.Filename) switch { - case ttyType == Tty: + case ttyType == TTY: db.createEntryLeader(pid, Terminal) - db.logger.Debugf("entry_eval %d: entry type is terminal", p.Pids.Tgid) + db.logger.Debugf("entry_eval %d: entry type is terminal", p.PIDs.Tgid) return &pid - case ttyType == TtyConsole && procBasename == "login": + case ttyType == TTYConsole && procBasename == "login": db.createEntryLeader(pid, EntryConsole) - db.logger.Debugf("entry_eval %d: entry type is console", p.Pids.Tgid) + db.logger.Debugf("entry_eval %d: entry type is console", p.PIDs.Tgid) return &pid - case p.Pids.Ppid == 1: + case p.PIDs.Ppid == 1: db.createEntryLeader(pid, Init) - db.logger.Debugf("entry_eval %d: entry type is init", p.Pids.Tgid) + db.logger.Debugf("entry_eval %d: entry type is init", p.PIDs.Tgid) return &pid case !isFilteredExecutable(procBasename): - if parent, ok := db.processes[p.Pids.Ppid]; ok { + if parent, ok := db.processes[p.PIDs.Ppid]; ok { parentBasename := basename(parent.Filename) if ttyType == Pts && parentBasename == "ssm-session-worker" { db.createEntryLeader(pid, Ssm) - db.logger.Debugf("entry_eval %d: entry type is ssm", p.Pids.Tgid) + db.logger.Debugf("entry_eval %d: entry type is ssm", p.PIDs.Tgid) return &pid } else if parentBasename == "sshd" && procBasename != "sshd" { // TODO: get ip from env vars db.createEntryLeader(pid, Sshd) - db.logger.Debugf("entry_eval %d: entry type is sshd", p.Pids.Tgid) + db.logger.Debugf("entry_eval %d: entry type is sshd", p.PIDs.Tgid) return &pid } else if isContainerRuntime(parentBasename) { db.createEntryLeader(pid, Container) - db.logger.Debugf("entry_eval %d: entry type is container", p.Pids.Tgid) + db.logger.Debugf("entry_eval %d: entry type is container", p.PIDs.Tgid) return &pid } } default: - db.logger.Debugf("entry_eval %d: is a filtered executable: %s", p.Pids.Tgid, procBasename) + db.logger.Debugf("entry_eval %d: is a filtered executable: %s", p.PIDs.Tgid, procBasename) } } @@ -401,15 +331,15 @@ func (db *DB) evaluateEntryLeader(p Process) *uint32 { name string }{ { - pid: p.Pids.Ppid, + pid: p.PIDs.Ppid, name: "parent", }, { - pid: p.Pids.Sid, + pid: p.PIDs.Sid, name: "session_leader", }, { - pid: p.Pids.Pgid, + pid: p.PIDs.Pgid, name: "group_leader", }, } @@ -417,22 +347,22 @@ func (db *DB) evaluateEntryLeader(p Process) *uint32 { for _, relation := range relations { if entry, ok := db.entryLeaderRelationships[relation.pid]; ok { entryType := db.entryLeaders[entry] - db.logger.Debugf("entry_eval %d: got entry_leader: %d (%s), from relative: %d (%s)", p.Pids.Tgid, entry, string(entryType), relation.pid, relation.name) + db.logger.Debugf("entry_eval %d: got entry_leader: %d (%s), from relative: %d (%s)", p.PIDs.Tgid, entry, string(entryType), relation.pid, relation.name) return &entry } else { - db.logger.Debugf("entry_eval %d: failed to find relative: %d (%s)", p.Pids.Tgid, relation.pid, relation.name) + db.logger.Debugf("entry_eval %d: failed to find relative: %d (%s)", p.PIDs.Tgid, relation.pid, relation.name) } } // if it's a session leader, then make it its own entry leader with unknown // entry type - if p.Pids.Tgid == p.Pids.Sid { + if p.PIDs.Tgid == p.PIDs.Sid { db.createEntryLeader(pid, EntryUnknown) - db.logger.Debugf("entry_eval %d: this is a session leader and no relative has an entry leader. entry type is unknown", p.Pids.Tgid) + db.logger.Debugf("entry_eval %d: this is a session leader and no relative has an entry leader. entry type is unknown", p.PIDs.Tgid) return &pid } - db.logger.Debugf("entry_eval %d: this is not a session leader and no relative has an entry leader, entry_leader will be unset", p.Pids.Tgid) + db.logger.Debugf("entry_eval %d: this is not a session leader and no relative has an entry leader, entry_leader will be unset", p.PIDs.Tgid) return nil } @@ -440,12 +370,12 @@ func (db *DB) InsertSetsid(setsid types.ProcessSetsidEvent) { db.Lock() defer db.Unlock() - if entry, ok := db.processes[setsid.Pids.Tgid]; ok { - entry.Pids = pidInfoFromProto(setsid.Pids) - db.processes[setsid.Pids.Tgid] = entry + if entry, ok := db.processes[setsid.PIDs.Tgid]; ok { + entry.PIDs = pidInfoFromProto(setsid.PIDs) + db.processes[setsid.PIDs.Tgid] = entry } else { - db.processes[setsid.Pids.Tgid] = Process{ - Pids: pidInfoFromProto(setsid.Pids), + db.processes[setsid.PIDs.Tgid] = Process{ + PIDs: pidInfoFromProto(setsid.PIDs), } } } @@ -454,36 +384,22 @@ func (db *DB) InsertExit(exit types.ProcessExitEvent) { db.Lock() defer db.Unlock() - pid := exit.Pids.Tgid + pid := exit.PIDs.Tgid delete(db.processes, pid) delete(db.entryLeaders, pid) delete(db.entryLeaderRelationships, pid) } -// TODO: is this the correct definition? I looked in endpoint and I swear it looks too simple/generalized -func interactiveFromTty(tty types.TtyDev) bool { - return TtyUnknown != getTtyType(tty.Major, tty.Minor) -} - -func ecsCapsFromU64(capabilities uint64) []string { - var ecsCaps []string - if c := bits.OnesCount64(capabilities); c > 0 { - ecsCaps = make([]string, 0, c) - } - for bitnum := 0; bitnum < 64; bitnum++ { - if (capabilities & (1 << bitnum)) > 0 { - ecsCaps = append(ecsCaps, capNames[bitnum]) - } - } - return ecsCaps +func interactiveFromTTY(tty types.TTYDev) bool { + return TTYUnknown != getTTYType(tty.Major, tty.Minor) } func fullProcessFromDBProcess(p Process) types.Process { - reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(p.Pids.StartTimeNs) - interactive := interactiveFromTty(p.CTty) + reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(p.PIDs.StartTimeNS) + interactive := interactiveFromTTY(p.CTTY) ret := types.Process{ - PID: p.Pids.Tgid, + PID: p.PIDs.Tgid, Start: timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime), Name: basename(p.Filename), Executable: p.Filename, @@ -496,19 +412,19 @@ func fullProcessFromDBProcess(p Process) types.Process { egid := p.Creds.Egid ret.User.ID = strconv.FormatUint(uint64(euid), 10) ret.Group.ID = strconv.FormatUint(uint64(egid), 10) - ret.Thread.Capabilities.Permitted = ecsCapsFromU64(p.Creds.CapPermitted) - ret.Thread.Capabilities.Effective = ecsCapsFromU64(p.Creds.CapEffective) + ret.Thread.Capabilities.Permitted, _ = capabilities.FromUint64(p.Creds.CapPermitted) + ret.Thread.Capabilities.Effective, _ = capabilities.FromUint64(p.Creds.CapEffective) return ret } func fillParent(process *types.Process, parent Process) { - reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(parent.Pids.StartTimeNs) + reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(parent.PIDs.StartTimeNS) - interactive := interactiveFromTty(parent.CTty) + interactive := interactiveFromTTY(parent.CTTY) euid := parent.Creds.Euid egid := parent.Creds.Egid - process.Parent.PID = parent.Pids.Tgid + process.Parent.PID = parent.PIDs.Tgid process.Parent.Start = timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime) process.Parent.Name = basename(parent.Filename) process.Parent.Executable = parent.Filename @@ -520,12 +436,12 @@ func fillParent(process *types.Process, parent Process) { } func fillGroupLeader(process *types.Process, groupLeader Process) { - reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(groupLeader.Pids.StartTimeNs) + reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(groupLeader.PIDs.StartTimeNS) - interactive := interactiveFromTty(groupLeader.CTty) + interactive := interactiveFromTTY(groupLeader.CTTY) euid := groupLeader.Creds.Euid egid := groupLeader.Creds.Egid - process.GroupLeader.PID = groupLeader.Pids.Tgid + process.GroupLeader.PID = groupLeader.PIDs.Tgid process.GroupLeader.Start = timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime) process.GroupLeader.Name = basename(groupLeader.Filename) process.GroupLeader.Executable = groupLeader.Filename @@ -537,12 +453,12 @@ func fillGroupLeader(process *types.Process, groupLeader Process) { } func fillSessionLeader(process *types.Process, sessionLeader Process) { - reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(sessionLeader.Pids.StartTimeNs) + reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(sessionLeader.PIDs.StartTimeNS) - interactive := interactiveFromTty(sessionLeader.CTty) + interactive := interactiveFromTTY(sessionLeader.CTTY) euid := sessionLeader.Creds.Euid egid := sessionLeader.Creds.Egid - process.SessionLeader.PID = sessionLeader.Pids.Tgid + process.SessionLeader.PID = sessionLeader.PIDs.Tgid process.SessionLeader.Start = timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime) process.SessionLeader.Name = basename(sessionLeader.Filename) process.SessionLeader.Executable = sessionLeader.Filename @@ -554,12 +470,12 @@ func fillSessionLeader(process *types.Process, sessionLeader Process) { } func fillEntryLeader(process *types.Process, entryType EntryType, entryLeader Process) { - reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(entryLeader.Pids.StartTimeNs) + reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(entryLeader.PIDs.StartTimeNS) - interactive := interactiveFromTty(entryLeader.CTty) + interactive := interactiveFromTTY(entryLeader.CTTY) euid := entryLeader.Creds.Euid egid := entryLeader.Creds.Egid - process.EntryLeader.PID = entryLeader.Pids.Tgid + process.EntryLeader.PID = entryLeader.PIDs.Tgid process.EntryLeader.Start = timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime) process.EntryLeader.Name = basename(entryLeader.Filename) process.EntryLeader.Executable = entryLeader.Filename @@ -622,24 +538,24 @@ func (db *DB) GetProcess(pid uint32) (types.Process, error) { ret := fullProcessFromDBProcess(process) - if parent, ok := db.processes[process.Pids.Ppid]; ok { + if parent, ok := db.processes[process.PIDs.Ppid]; ok { fillParent(&ret, parent) } - if groupLeader, ok := db.processes[process.Pids.Pgid]; ok { + if groupLeader, ok := db.processes[process.PIDs.Pgid]; ok { fillGroupLeader(&ret, groupLeader) } - if sessionLeader, ok := db.processes[process.Pids.Sid]; ok { + if sessionLeader, ok := db.processes[process.PIDs.Sid]; ok { fillSessionLeader(&ret, sessionLeader) } - if entryLeaderPid, foundEntryLeaderPid := db.entryLeaderRelationships[process.Pids.Tgid]; foundEntryLeaderPid { - if entryLeader, foundEntryLeader := db.processes[entryLeaderPid]; foundEntryLeader { + if entryLeaderPID, foundEntryLeaderPID := db.entryLeaderRelationships[process.PIDs.Tgid]; foundEntryLeaderPID { + if entryLeader, foundEntryLeader := db.processes[entryLeaderPID]; foundEntryLeader { // if there is an entry leader then there is a matching member in the entryLeaders table - fillEntryLeader(&ret, db.entryLeaders[entryLeaderPid], entryLeader) + fillEntryLeader(&ret, db.entryLeaders[entryLeaderPID], entryLeader) } else { - db.logger.Errorf("failed to find entry leader entry %d for %d (%s)", entryLeaderPid, pid, db.processes[pid].Filename) + db.logger.Errorf("failed to find entry leader entry %d for %d (%s)", entryLeaderPID, pid, db.processes[pid].Filename) } } else { db.logger.Errorf("failed to find entry leader for %d (%s)", pid, db.processes[pid].Filename) @@ -674,17 +590,17 @@ func (db *DB) ScrapeProcfs() []uint32 { // sorting the slice to make sure that parents, session leaders, group // leaders come first in the queue sort.Slice(procs, func(i, j int) bool { - return procs[i].Pids.Tgid == procs[j].Pids.Ppid || - procs[i].Pids.Tgid == procs[j].Pids.Sid || - procs[i].Pids.Tgid == procs[j].Pids.Pgid + return procs[i].PIDs.Tgid == procs[j].PIDs.Ppid || + procs[i].PIDs.Tgid == procs[j].PIDs.Sid || + procs[i].PIDs.Tgid == procs[j].PIDs.Pgid }) pids := make([]uint32, 0) for _, procInfo := range procs { process := Process{ - Pids: pidInfoFromProto(procInfo.Pids), + PIDs: pidInfoFromProto(procInfo.PIDs), Creds: credInfoFromProto(procInfo.Creds), - CTty: ttyDevFromProto(procInfo.CTty), + CTTY: ttyDevFromProto(procInfo.CTTY), Argv: procInfo.Argv, Cwd: procInfo.Cwd, Env: procInfo.Env, @@ -692,7 +608,7 @@ func (db *DB) ScrapeProcfs() []uint32 { } db.insertProcess(process) - pids = append(pids, process.Pids.Tgid) + pids = append(pids, process.PIDs.Tgid) } return pids @@ -718,18 +634,18 @@ func isFilteredExecutable(executable string) bool { return stringStartsWithEntryInList(executable, filteredExecutables[:]) } -func getTtyType(major uint16, minor uint16) TtyType { +func getTTYType(major uint16, minor uint16) TTYType { if major >= ptsMinMajor && major <= ptsMaxMajor { return Pts } if ttyMajor == major { if minor <= consoleMaxMinor { - return TtyConsole + return TTYConsole } else if minor > consoleMaxMinor && minor <= ttyMaxMinor { - return Tty + return TTY } } - return TtyUnknown + return TTYUnknown } diff --git a/x-pack/auditbeat/processors/sessionmd/processdb/db_test.go b/x-pack/auditbeat/processors/sessionmd/processdb/db_test.go index 9e283d45e9ee..5e8001a68e54 100644 --- a/x-pack/auditbeat/processors/sessionmd/processdb/db_test.go +++ b/x-pack/auditbeat/processors/sessionmd/processdb/db_test.go @@ -10,45 +10,15 @@ import ( "testing" "github.com/stretchr/testify/assert" - "golang.org/x/sys/unix" "github.com/elastic/elastic-agent-libs/logp" ) var logger = logp.NewLogger("processdb") -func TestGetTtyType(t *testing.T) { - assert.Equal(t, TtyConsole, getTtyType(4, 0)) - assert.Equal(t, Pts, getTtyType(136, 0)) - assert.Equal(t, Tty, getTtyType(4, 64)) - assert.Equal(t, TtyUnknown, getTtyType(1000, 1000)) -} - -func TestCapsFromU64ToECS(t *testing.T) { - expected := []string{"CAP_CHOWN"} - assert.Equal(t, expected, ecsCapsFromU64(uint64(1<> 8) & 0xf) } -func MinorTty(ttyNr uint32) uint16 { +func MinorTTY(ttyNr uint32) uint16 { return uint16(((ttyNr & 0xfff00000) >> 20) | (ttyNr & 0xff)) } @@ -46,9 +46,9 @@ func NewProcfsReader(logger logp.Logger) ProcfsReader { type Stat procfs.ProcStat type ProcessInfo struct { - Pids types.PidInfo + PIDs types.PIDInfo Creds types.CredInfo - CTty types.TtyDev + CTTY types.TTYDev Argv []string Cwd string Env map[string]string @@ -179,8 +179,8 @@ func (r ProcfsReader) getProcessInfo(proc procfs.Proc) (ProcessInfo, error) { startTimeNs := timeutils.TicksToNs(stat.Starttime) return ProcessInfo{ - Pids: types.PidInfo{ - StartTimeNs: startTimeNs, + PIDs: types.PIDInfo{ + StartTimeNS: startTimeNs, Tid: pid, Tgid: pid, Ppid: uint32(stat.PPID), @@ -188,9 +188,9 @@ func (r ProcfsReader) getProcessInfo(proc procfs.Proc) (ProcessInfo, error) { Sid: uint32(stat.Session), }, Creds: creds, - CTty: types.TtyDev{ - Major: MajorTty(uint32(stat.TTY)), - Minor: MinorTty(uint32(stat.TTY)), + CTTY: types.TTYDev{ + Major: MajorTTY(uint32(stat.TTY)), + Minor: MinorTTY(uint32(stat.TTY)), }, Cwd: cwd, Argv: argv, diff --git a/x-pack/auditbeat/processors/sessionmd/provider/ebpf_provider/ebpf_provider.go b/x-pack/auditbeat/processors/sessionmd/provider/ebpf_provider/ebpf_provider.go index 15c180068a53..2b9b540e037c 100644 --- a/x-pack/auditbeat/processors/sessionmd/provider/ebpf_provider/ebpf_provider.go +++ b/x-pack/auditbeat/processors/sessionmd/provider/ebpf_provider/ebpf_provider.go @@ -63,21 +63,21 @@ func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (pr continue } pe := types.ProcessForkEvent{ - ParentPids: types.PidInfo{ + ParentPIDs: types.PIDInfo{ Tid: body.ParentPids.Tid, Tgid: body.ParentPids.Tgid, Ppid: body.ParentPids.Ppid, Pgid: body.ParentPids.Pgid, Sid: body.ParentPids.Sid, - StartTimeNs: body.ParentPids.StartTimeNs, + StartTimeNS: body.ParentPids.StartTimeNs, }, - ChildPids: types.PidInfo{ + ChildPIDs: types.PIDInfo{ Tid: body.ChildPids.Tid, Tgid: body.ChildPids.Tgid, Ppid: body.ChildPids.Ppid, Pgid: body.ChildPids.Pgid, Sid: body.ChildPids.Sid, - StartTimeNs: body.ChildPids.StartTimeNs, + StartTimeNS: body.ChildPids.StartTimeNs, }, Creds: types.CredInfo{ Ruid: body.Creds.Ruid, @@ -98,13 +98,13 @@ func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (pr continue } pe := types.ProcessExecEvent{ - Pids: types.PidInfo{ + PIDs: types.PIDInfo{ Tid: body.Pids.Tid, Tgid: body.Pids.Tgid, Ppid: body.Pids.Ppid, Pgid: body.Pids.Pgid, Sid: body.Pids.Sid, - StartTimeNs: body.Pids.StartTimeNs, + StartTimeNS: body.Pids.StartTimeNs, }, Creds: types.CredInfo{ Ruid: body.Creds.Ruid, @@ -116,11 +116,11 @@ func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (pr CapPermitted: body.Creds.CapPermitted, CapEffective: body.Creds.CapEffective, }, - CTty: types.TtyDev{ + CTTY: types.TTYDev{ Major: body.CTTY.Major, Minor: body.CTTY.Minor, }, - Cwd: body.Cwd, + CWD: body.Cwd, Argv: body.Argv, Env: body.Env, Filename: body.Filename, @@ -133,13 +133,13 @@ func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (pr continue } pe := types.ProcessExitEvent{ - Pids: types.PidInfo{ + PIDs: types.PIDInfo{ Tid: body.Pids.Tid, Tgid: body.Pids.Tgid, Ppid: body.Pids.Ppid, Pgid: body.Pids.Pgid, Sid: body.Pids.Sid, - StartTimeNs: body.Pids.StartTimeNs, + StartTimeNS: body.Pids.StartTimeNs, }, ExitCode: body.ExitCode, } diff --git a/x-pack/auditbeat/processors/sessionmd/types/events.go b/x-pack/auditbeat/processors/sessionmd/types/events.go index dd3daaabc988..5f8d67d763f1 100644 --- a/x-pack/auditbeat/processors/sessionmd/types/events.go +++ b/x-pack/auditbeat/processors/sessionmd/types/events.go @@ -20,14 +20,14 @@ type ( ) const ( - Cwd Field = iota + 1 + CWD Field = iota + 1 Argv Env Filename ) -type PidInfo struct { - StartTimeNs uint64 +type PIDInfo struct { + StartTimeNS uint64 Tid uint32 Tgid uint32 Vpid uint32 @@ -47,48 +47,48 @@ type CredInfo struct { CapEffective uint64 } -type TtyWinsize struct { +type TTYWinsize struct { Rows uint16 Cols uint16 } -type TtyTermios struct { +type TTYTermios struct { CIflag uint32 COflag uint32 CLflag uint32 CCflag uint32 } -type TtyDev struct { +type TTYDev struct { Minor uint16 Major uint16 - Winsize TtyWinsize - Termios TtyTermios + Winsize TTYWinsize + Termios TTYTermios } type ProcessForkEvent struct { - ParentPids PidInfo - ChildPids PidInfo + ParentPIDs PIDInfo + ChildPIDs PIDInfo Creds CredInfo } type ProcessExecEvent struct { - Pids PidInfo + PIDs PIDInfo Creds CredInfo - CTty TtyDev + CTTY TTYDev // varlen fields - Cwd string + CWD string Argv []string Env map[string]string Filename string } type ProcessExitEvent struct { - Pids PidInfo + PIDs PIDInfo ExitCode int32 } type ProcessSetsidEvent struct { - Pids PidInfo + PIDs PIDInfo } From f2443cd4ecae73ab554b07bd9de51384a4bdd2b1 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Wed, 21 Feb 2024 18:53:30 -0800 Subject: [PATCH 29/32] Change some struct member visibility --- .../sessionmd/add_session_metadata.go | 6 ++-- .../sessionmd/add_session_metadata_test.go | 10 +++---- .../auditbeat/processors/sessionmd/config.go | 6 ++-- .../processors/sessionmd/processdb/db.go | 30 +++++++++---------- 4 files changed, 26 insertions(+), 26 deletions(-) diff --git a/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go b/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go index 231527b88ded..c7423a47aab7 100644 --- a/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go +++ b/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go @@ -19,7 +19,7 @@ import ( "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/procfs" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider" "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider/ebpf_provider" - "github.com/elastic/elastic-agent-libs/config" + cfg "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" ) @@ -34,13 +34,13 @@ func init() { } type addSessionMetadata struct { - config Config + config config logger *logp.Logger db *processdb.DB provider provider.Provider } -func New(cfg *config.C) (beat.Processor, error) { +func New(cfg *cfg.C) (beat.Processor, error) { c := defaultConfig() if err := cfg.Unpack(&c); err != nil { return nil, fmt.Errorf("fail to unpack the %v configuration: %w", processorName, err) diff --git a/x-pack/auditbeat/processors/sessionmd/add_session_metadata_test.go b/x-pack/auditbeat/processors/sessionmd/add_session_metadata_test.go index c8b0e5731565..2f00b2f419d0 100644 --- a/x-pack/auditbeat/processors/sessionmd/add_session_metadata_test.go +++ b/x-pack/auditbeat/processors/sessionmd/add_session_metadata_test.go @@ -24,14 +24,14 @@ var ( enrichTests = []struct { testName string mockProcesses []types.ProcessExecEvent - config Config + config config input beat.Event expected beat.Event expect_error bool }{ { testName: "Enrich Process", - config: Config{ + config: config{ ReplaceFields: false, PIDField: "process.pid", }, @@ -93,7 +93,7 @@ var ( }, { testName: "No PID Field in Event", - config: Config{ + config: config{ ReplaceFields: false, PIDField: "process.pid", }, @@ -112,7 +112,7 @@ var ( }, { testName: "PID Not Number", - config: Config{ + config: config{ ReplaceFields: false, PIDField: "process.pid", }, @@ -132,7 +132,7 @@ var ( }, { testName: "PID not in DB", - config: Config{ + config: config{ ReplaceFields: false, PIDField: "process.pid", }, diff --git a/x-pack/auditbeat/processors/sessionmd/config.go b/x-pack/auditbeat/processors/sessionmd/config.go index a082c7d1fd12..31c07c9065f1 100644 --- a/x-pack/auditbeat/processors/sessionmd/config.go +++ b/x-pack/auditbeat/processors/sessionmd/config.go @@ -7,14 +7,14 @@ package sessionmd // Config for add_session_metadata processor. -type Config struct { +type config struct { Backend string `config:"backend"` ReplaceFields bool `config:"replace_fields"` PIDField string `config:"pid_field"` } -func defaultConfig() Config { - return Config{ +func defaultConfig() config { + return config{ Backend: "auto", ReplaceFields: false, PIDField: "process.pid", diff --git a/x-pack/auditbeat/processors/sessionmd/processdb/db.go b/x-pack/auditbeat/processors/sessionmd/processdb/db.go index 4b0b90ea0046..309fb018f9a8 100644 --- a/x-pack/auditbeat/processors/sessionmd/processdb/db.go +++ b/x-pack/auditbeat/processors/sessionmd/processdb/db.go @@ -165,7 +165,7 @@ func ttyDevFromProto(p types.TTYDev) types.TTYDev { } type DB struct { - sync.RWMutex + mutex sync.RWMutex logger *logp.Logger processes map[uint32]Process entryLeaders map[uint32]EntryType @@ -207,8 +207,8 @@ func basename(pathStr string) string { } func (db *DB) InsertFork(fork types.ProcessForkEvent) { - db.Lock() - defer db.Unlock() + db.mutex.Lock() + defer db.mutex.Unlock() pid := fork.ChildPIDs.Tgid ppid := fork.ParentPIDs.Tgid @@ -240,8 +240,8 @@ func (db *DB) insertProcess(process Process) { } func (db *DB) InsertExec(exec types.ProcessExecEvent) { - db.Lock() - defer db.Unlock() + db.mutex.Lock() + defer db.mutex.Unlock() proc := Process{ PIDs: pidInfoFromProto(exec.PIDs), @@ -367,8 +367,8 @@ func (db *DB) evaluateEntryLeader(p Process) *uint32 { } func (db *DB) InsertSetsid(setsid types.ProcessSetsidEvent) { - db.Lock() - defer db.Unlock() + db.mutex.Lock() + defer db.mutex.Unlock() if entry, ok := db.processes[setsid.PIDs.Tgid]; ok { entry.PIDs = pidInfoFromProto(setsid.PIDs) @@ -381,8 +381,8 @@ func (db *DB) InsertSetsid(setsid types.ProcessSetsidEvent) { } func (db *DB) InsertExit(exit types.ProcessExitEvent) { - db.Lock() - defer db.Unlock() + db.mutex.Lock() + defer db.mutex.Unlock() pid := exit.PIDs.Tgid delete(db.processes, pid) @@ -528,8 +528,8 @@ func setSameAsProcess(process *types.Process) { } func (db *DB) GetProcess(pid uint32) (types.Process, error) { - db.RLock() - defer db.RUnlock() + db.mutex.RLock() + defer db.mutex.RUnlock() process, ok := db.processes[pid] if !ok { @@ -568,8 +568,8 @@ func (db *DB) GetProcess(pid uint32) (types.Process, error) { } func (db *DB) GetEntryType(pid uint32) (EntryType, error) { - db.RLock() - defer db.RUnlock() + db.mutex.RLock() + defer db.mutex.RUnlock() if entryType, ok := db.entryLeaders[pid]; ok { return entryType, nil @@ -578,8 +578,8 @@ func (db *DB) GetEntryType(pid uint32) (EntryType, error) { } func (db *DB) ScrapeProcfs() []uint32 { - db.Lock() - defer db.Unlock() + db.mutex.Lock() + defer db.mutex.Unlock() procs, err := db.procfs.GetAllProcesses() if err != nil { From 36c899817c00af932e0e09446c701b9ccccab62b Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Thu, 14 Mar 2024 08:54:04 -0700 Subject: [PATCH 30/32] Remove possibilities of panics Remove possibe panics in program initialization, and handle unexpected events more gracefully. --- .../sessionmd/add_session_metadata.go | 26 ++++- .../sessionmd/add_session_metadata_test.go | 107 +++++++++++++++++- .../processors/sessionmd/processdb/db.go | 37 ++++-- .../sessionmd/processdb/entry_leader_test.go | 57 ++++++---- .../processors/sessionmd/timeutils/time.go | 47 ++++++-- 5 files changed, 230 insertions(+), 44 deletions(-) diff --git a/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go b/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go index c7423a47aab7..2143818ae830 100644 --- a/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go +++ b/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go @@ -50,7 +50,10 @@ func New(cfg *cfg.C) (beat.Processor, error) { ctx := context.Background() reader := procfs.NewProcfsReader(*logger) - db := processdb.NewDB(reader, *logger) + db, err := processdb.NewDB(reader, *logger) + if err != nil { + return nil, fmt.Errorf("failed to create DB: %w", err) + } backfilledPIDs := db.ScrapeProcfs() logger.Debugf("backfilled %d processes", len(backfilledPIDs)) @@ -118,7 +121,15 @@ func (p *addSessionMetadata) enrich(ev *beat.Event) (*beat.Event, error) { processMap := fullProcess.ToMap() - err = mapstr.MergeFieldsDeep(result.Fields["process"].(mapstr.M), processMap, true) + if b, err := result.Fields.HasKey("process"); !b || err != nil { + return nil, fmt.Errorf("no process field in event") + } + m, ok := tryToMapStr(result.Fields["process"]) + if !ok { + return nil, fmt.Errorf("process field type not supported") + } + + err = mapstr.MergeFieldsDeep(m, processMap, true) if err != nil { return nil, fmt.Errorf("merging enriched fields with event: %w", err) } @@ -191,3 +202,14 @@ func (p *addSessionMetadata) replaceFields(ev *beat.Event) error { } return nil } + +func tryToMapStr(v interface{}) (mapstr.M, bool) { + switch m := v.(type) { + case mapstr.M: + return m, true + case map[string]interface{}: + return mapstr.M(m), true + default: + return nil, false + } +} diff --git a/x-pack/auditbeat/processors/sessionmd/add_session_metadata_test.go b/x-pack/auditbeat/processors/sessionmd/add_session_metadata_test.go index 2f00b2f419d0..4890505aac48 100644 --- a/x-pack/auditbeat/processors/sessionmd/add_session_metadata_test.go +++ b/x-pack/auditbeat/processors/sessionmd/add_session_metadata_test.go @@ -30,7 +30,7 @@ var ( expect_error bool }{ { - testName: "Enrich Process", + testName: "enrich process", config: config{ ReplaceFields: false, PIDField: "process.pid", @@ -92,7 +92,7 @@ var ( expect_error: false, }, { - testName: "No PID Field in Event", + testName: "no PID field in event", config: config{ ReplaceFields: false, PIDField: "process.pid", @@ -111,7 +111,7 @@ var ( expect_error: true, }, { - testName: "PID Not Number", + testName: "PID not number", config: config{ ReplaceFields: false, PIDField: "process.pid", @@ -150,6 +150,104 @@ var ( }, expect_error: true, }, + { + testName: "process field not in event", + // This event, without a "process" field, is not supported by enrich, it should be handled gracefully + config: config{ + ReplaceFields: false, + PIDField: "action.pid", + }, + input: beat.Event{ + Fields: mapstr.M{ + "action": mapstr.M{ + "pid": "1010", + }, + }, + }, + expect_error: true, + }, + { + testName: "process field not mapstr", + // Unsupported process field type should be handled gracefully + config: config{ + ReplaceFields: false, + PIDField: "action.pid", + }, + input: beat.Event{ + Fields: mapstr.M{ + "action": mapstr.M{ + "pid": "100", + }, + "process": map[int]int{ + 10: 100, + 20: 200, + }, + }, + }, + expect_error: true, + }, + { + testName: "enrich event with map[string]any process field", + config: config{ + ReplaceFields: false, + PIDField: "process.pid", + }, + mockProcesses: []types.ProcessExecEvent{ + { + PIDs: types.PIDInfo{ + Tid: uint32(100), + Tgid: uint32(100), + Ppid: uint32(50), + Pgid: uint32(100), + Sid: uint32(40), + }, + CWD: "/", + Filename: "/bin/ls", + }, + { + PIDs: types.PIDInfo{ + Tid: uint32(50), + Tgid: uint32(50), + Ppid: uint32(40), + Sid: uint32(40), + }, + }, + { + PIDs: types.PIDInfo{ + Tid: uint32(40), + Tgid: uint32(40), + Ppid: uint32(1), + Sid: uint32(1), + }, + }, + }, + input: beat.Event{ + Fields: map[string]any{ + "process": map[string]any{ + "pid": uint32(100), + }, + }, + }, + expected: beat.Event{ + Fields: mapstr.M{ + "process": mapstr.M{ + "executable": "/bin/ls", + "working_directory": "/", + "pid": uint32(100), + "parent": mapstr.M{ + "pid": uint32(50), + }, + "session_leader": mapstr.M{ + "pid": uint32(40), + }, + "group_leader": mapstr.M{ + "pid": uint32(100), + }, + }, + }, + }, + expect_error: false, + }, } filterTests = []struct { @@ -228,7 +326,8 @@ var ( func TestEnrich(t *testing.T) { for _, tt := range enrichTests { reader := procfs.NewMockReader() - db := processdb.NewDB(reader, *logger) + db, err := processdb.NewDB(reader, *logger) + assert.Nil(t, err) for _, ev := range tt.mockProcesses { db.InsertExec(ev) diff --git a/x-pack/auditbeat/processors/sessionmd/processdb/db.go b/x-pack/auditbeat/processors/sessionmd/processdb/db.go index 309fb018f9a8..6b2de897973b 100644 --- a/x-pack/auditbeat/processors/sessionmd/processdb/db.go +++ b/x-pack/auditbeat/processors/sessionmd/processdb/db.go @@ -84,22 +84,22 @@ type Process struct { } var ( - // The contents of these two files are needed to calculate entity IDs. - // Fail fast on startup if we can't read them. - bootID = mustReadBootID() - pidNsInode = mustReadPIDNsInode() + bootID string + pidNsInode uint64 + initError error + once sync.Once ) -func mustReadBootID() string { +func readBootID() (string, error) { bootID, err := os.ReadFile("/proc/sys/kernel/random/boot_id") if err != nil { panic(fmt.Sprintf("could not read /proc/sys/kernel/random/boot_id: %v", err)) } - return strings.TrimRight(string(bootID), "\n") + return strings.TrimRight(string(bootID), "\n"), nil } -func mustReadPIDNsInode() uint64 { +func readPIDNsInode() (uint64, error) { var ret uint64 pidNsInodeRaw, err := os.Readlink("/proc/self/ns/pid") @@ -111,7 +111,7 @@ func mustReadPIDNsInode() uint64 { panic(fmt.Sprintf("could not parse contents of /proc/self/ns/pid (%s): %v", pidNsInodeRaw, err)) } - return ret + return ret, nil } func pidInfoFromProto(p types.PIDInfo) types.PIDInfo { @@ -164,6 +164,19 @@ func ttyDevFromProto(p types.TTYDev) types.TTYDev { } } +func initialize() { + var err error + bootID, err = readBootID() + if err != nil { + initError = err + return + } + pidNsInode, err = readPIDNsInode() + if err != nil { + initError = err + } +} + type DB struct { mutex sync.RWMutex logger *logp.Logger @@ -173,14 +186,18 @@ type DB struct { procfs procfs.Reader } -func NewDB(reader procfs.Reader, logger logp.Logger) *DB { +func NewDB(reader procfs.Reader, logger logp.Logger) (*DB, error) { + once.Do(initialize) + if initError != nil { + return &DB{}, initError + } return &DB{ logger: logp.NewLogger("processdb"), processes: make(map[uint32]Process), entryLeaders: make(map[uint32]EntryType), entryLeaderRelationships: make(map[uint32]uint32), procfs: reader, - } + }, nil } func (db *DB) calculateEntityIDv1(pid uint32, startTime time.Time) string { diff --git a/x-pack/auditbeat/processors/sessionmd/processdb/entry_leader_test.go b/x-pack/auditbeat/processors/sessionmd/processdb/entry_leader_test.go index 06d1ff5ee643..15f98250f55d 100644 --- a/x-pack/auditbeat/processors/sessionmd/processdb/entry_leader_test.go +++ b/x-pack/auditbeat/processors/sessionmd/processdb/entry_leader_test.go @@ -188,7 +188,8 @@ func populateProcfsWithInit(reader *procfs.MockReader) { func TestSingleProcessSessionLeaderEntryTypeTerminal(t *testing.T) { reader := procfs.NewMockReader() - db := NewDB(reader, *logger) + db, err := NewDB(reader, *logger) + require.Nil(t, err) db.ScrapeProcfs() pid := uint32(1234) @@ -211,7 +212,8 @@ func TestSingleProcessSessionLeaderEntryTypeTerminal(t *testing.T) { func TestSingleProcessSessionLeaderLoginProcess(t *testing.T) { reader := procfs.NewMockReader() - db := NewDB(reader, *logger) + db, err := NewDB(reader, *logger) + require.Nil(t, err) db.ScrapeProcfs() pid := uint32(1234) @@ -239,7 +241,8 @@ func TestSingleProcessSessionLeaderLoginProcess(t *testing.T) { func TestSingleProcessSessionLeaderChildOfInit(t *testing.T) { reader := procfs.NewMockReader() - db := NewDB(reader, *logger) + db, err := NewDB(reader, *logger) + require.Nil(t, err) db.ScrapeProcfs() pid := uint32(100) @@ -268,7 +271,8 @@ func TestSingleProcessSessionLeaderChildOfInit(t *testing.T) { func TestSingleProcessSessionLeaderChildOfSsmSessionWorker(t *testing.T) { reader := procfs.NewMockReader() - db := NewDB(reader, *logger) + db, err := NewDB(reader, *logger) + require.Nil(t, err) db.ScrapeProcfs() ssmPID := uint32(999) @@ -303,7 +307,8 @@ func TestSingleProcessSessionLeaderChildOfSsmSessionWorker(t *testing.T) { func TestSingleProcessSessionLeaderChildOfSshd(t *testing.T) { reader := procfs.NewMockReader() - db := NewDB(reader, *logger) + db, err := NewDB(reader, *logger) + require.Nil(t, err) db.ScrapeProcfs() sshdPID := uint32(999) @@ -337,7 +342,8 @@ func TestSingleProcessSessionLeaderChildOfSshd(t *testing.T) { func TestSingleProcessSessionLeaderChildOfContainerdShim(t *testing.T) { reader := procfs.NewMockReader() - db := NewDB(reader, *logger) + db, err := NewDB(reader, *logger) + require.Nil(t, err) db.ScrapeProcfs() containerdShimPID := uint32(999) @@ -371,7 +377,8 @@ func TestSingleProcessSessionLeaderChildOfContainerdShim(t *testing.T) { func TestSingleProcessSessionLeaderChildOfRunc(t *testing.T) { reader := procfs.NewMockReader() - db := NewDB(reader, *logger) + db, err := NewDB(reader, *logger) + require.Nil(t, err) db.ScrapeProcfs() runcPID := uint32(999) @@ -406,7 +413,8 @@ func TestSingleProcessSessionLeaderChildOfRunc(t *testing.T) { func TestSingleProcessEmptyProcess(t *testing.T) { reader := procfs.NewMockReader() - db := NewDB(reader, *logger) + db, err := NewDB(reader, *logger) + require.Nil(t, err) db.ScrapeProcfs() // No information in proc at all, entry type should be "unknown" @@ -438,7 +446,8 @@ func TestSingleProcessEmptyProcess(t *testing.T) { // EntryLeaderEntryMetaType func TestSingleProcessOverwriteOldEntryLeader(t *testing.T) { reader := procfs.NewMockReader() - db := NewDB(reader, *logger) + db, err := NewDB(reader, *logger) + require.Nil(t, err) db.ScrapeProcfs() ssmPID := uint32(999) @@ -508,7 +517,8 @@ func TestSingleProcessOverwriteOldEntryLeader(t *testing.T) { func TestInitSshdBashLs(t *testing.T) { reader := procfs.NewMockReader() populateProcfsWithInit(reader) - db := NewDB(reader, *logger) + db, err := NewDB(reader, *logger) + require.Nil(t, err) db.ScrapeProcfs() sshdPID := uint32(100) @@ -590,7 +600,8 @@ func TestInitSshdBashLs(t *testing.T) { func TestInitSshdSshdBashLs(t *testing.T) { reader := procfs.NewMockReader() populateProcfsWithInit(reader) - db := NewDB(reader, *logger) + db, err := NewDB(reader, *logger) + require.Nil(t, err) db.ScrapeProcfs() sshd0PID := uint32(100) @@ -681,7 +692,8 @@ func TestInitSshdSshdBashLs(t *testing.T) { func TestInitSshdSshdSshdBashLs(t *testing.T) { reader := procfs.NewMockReader() populateProcfsWithInit(reader) - db := NewDB(reader, *logger) + db, err := NewDB(reader, *logger) + require.Nil(t, err) db.ScrapeProcfs() sshd0PID := uint32(100) @@ -789,7 +801,8 @@ func TestInitSshdSshdSshdBashLs(t *testing.T) { func TestInitContainerdContainerdShim(t *testing.T) { reader := procfs.NewMockReader() populateProcfsWithInit(reader) - db := NewDB(reader, *logger) + db, err := NewDB(reader, *logger) + require.Nil(t, err) db.ScrapeProcfs() containerdPID := uint32(100) @@ -841,7 +854,8 @@ func TestInitContainerdContainerdShim(t *testing.T) { func TestInitContainerdShimBashContainerdShimIsReparentedToInit(t *testing.T) { reader := procfs.NewMockReader() populateProcfsWithInit(reader) - db := NewDB(reader, *logger) + db, err := NewDB(reader, *logger) + require.Nil(t, err) db.ScrapeProcfs() containerdPID := uint32(100) @@ -909,7 +923,8 @@ func TestInitContainerdShimBashContainerdShimIsReparentedToInit(t *testing.T) { func TestInitContainerdShimPauseContainerdShimIsReparentedToInit(t *testing.T) { reader := procfs.NewMockReader() populateProcfsWithInit(reader) - db := NewDB(reader, *logger) + db, err := NewDB(reader, *logger) + require.Nil(t, err) db.ScrapeProcfs() containerdPID := uint32(100) @@ -980,7 +995,8 @@ func TestInitContainerdShimPauseContainerdShimIsReparentedToInit(t *testing.T) { func TestInitSshdBashLsAndGrepGrepOnlyHasGroupLeader(t *testing.T) { reader := procfs.NewMockReader() populateProcfsWithInit(reader) - db := NewDB(reader, *logger) + db, err := NewDB(reader, *logger) + require.Nil(t, err) db.ScrapeProcfs() sshdPID := uint32(100) @@ -1066,7 +1082,8 @@ func TestInitSshdBashLsAndGrepGrepOnlyHasGroupLeader(t *testing.T) { func TestInitSshdBashLsAndGrepGrepOnlyHasSessionLeader(t *testing.T) { reader := procfs.NewMockReader() populateProcfsWithInit(reader) - db := NewDB(reader, *logger) + db, err := NewDB(reader, *logger) + require.Nil(t, err) db.ScrapeProcfs() sshdPID := uint32(100) @@ -1147,7 +1164,8 @@ func TestInitSshdBashLsAndGrepGrepOnlyHasSessionLeader(t *testing.T) { // entry meta type of "unknown" and making it an entry leader. func TestGrepInIsolation(t *testing.T) { reader := procfs.NewMockReader() - db := NewDB(reader, *logger) + db, err := NewDB(reader, *logger) + require.Nil(t, err) db.ScrapeProcfs() grepPID := uint32(1001) @@ -1179,7 +1197,8 @@ func TestGrepInIsolation(t *testing.T) { // Kernel threads should never have an entry meta type or entry leader set. func TestKernelThreads(t *testing.T) { reader := procfs.NewMockReader() - db := NewDB(reader, *logger) + db, err := NewDB(reader, *logger) + require.Nil(t, err) kthreaddPID := uint32(2) rcuGpPID := uint32(3) diff --git a/x-pack/auditbeat/processors/sessionmd/timeutils/time.go b/x-pack/auditbeat/processors/sessionmd/timeutils/time.go index 5328717b99b6..15c62be90084 100644 --- a/x-pack/auditbeat/processors/sessionmd/timeutils/time.go +++ b/x-pack/auditbeat/processors/sessionmd/timeutils/time.go @@ -8,6 +8,7 @@ package timeutils import ( "fmt" + "sync" "time" "github.com/prometheus/procfs" @@ -15,36 +16,60 @@ import ( ) var ( - bootTime = mustGetBootTime() - ticksPerSecond = mustGetTicksPerSecond() + bootTime time.Time + ticksPerSecond uint64 + initError error + once sync.Once ) -func mustGetBootTime() time.Time { +func initialize() { + var err error + bootTime, err = getBootTime() + if err != nil { + initError = err + return + } + + ticksPerSecond, err = getTicksPerSecond() + if err != nil { + initError = err + } +} + +func getBootTime() (time.Time, error) { fs, err := procfs.NewDefaultFS() if err != nil { - panic(fmt.Sprintf("could not get procfs: %v", err)) + return time.Time{}, fmt.Errorf("could not get procfs: %w", err) } stat, err := fs.Stat() if err != nil { - panic(fmt.Sprintf("could not read /proc/stat: %v", err)) + return time.Time{}, fmt.Errorf("could not read /proc/stat: %w", err) } - return time.Unix(int64(stat.BootTime), 0) + return time.Unix(int64(stat.BootTime), 0), nil } -func mustGetTicksPerSecond() uint64 { +func getTicksPerSecond() (uint64, error) { tps, err := sysconf.Sysconf(sysconf.SC_CLK_TCK) if err != nil { - panic(fmt.Sprintf("sysconf(SC_CLK_TCK) failed: %v", err)) + return 0, fmt.Errorf("sysconf(SC_CLK_TCK) failed: %w", err) } - return uint64(tps) + return uint64(tps), nil } func TicksToNs(ticks uint64) uint64 { + once.Do(initialize) + if initError != nil { + return 0 + } return ticks * uint64(time.Second.Nanoseconds()) / ticksPerSecond } func TimeFromNsSinceBoot(t time.Duration) *time.Time { + once.Do(initialize) + if initError != nil { + return &time.Time{} + } timestamp := bootTime.Add(t) return ×tamp } @@ -60,5 +85,9 @@ func TimeFromNsSinceBoot(t time.Duration) *time.Time { // - We store timestamps as nanoseconds, but reduce the precision to 1/100th // second func ReduceTimestampPrecision(timeNs uint64) time.Duration { + once.Do(initialize) + if initError != nil { + return 0 + } return time.Duration(timeNs).Truncate(time.Second / time.Duration(ticksPerSecond)) } From 52e2b6010911cb6d2fb2f241e9362050238f40f0 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Thu, 28 Mar 2024 16:53:56 -0700 Subject: [PATCH 31/32] Fix up more code * Move event to after some conditions are checked * Use sync.OnceValues in time utils * Add issue reference in comment --- .../sessionmd/add_session_metadata.go | 10 +++--- .../processors/sessionmd/timeutils/time.go | 34 +++++-------------- 2 files changed, 14 insertions(+), 30 deletions(-) diff --git a/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go b/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go index 2143818ae830..50636f9d476c 100644 --- a/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go +++ b/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go @@ -117,22 +117,22 @@ func (p *addSessionMetadata) enrich(ev *beat.Event) (*beat.Event, error) { return nil, fmt.Errorf("pid %v not found in db: %w", pid, err) } - result := ev.Clone() - processMap := fullProcess.ToMap() - if b, err := result.Fields.HasKey("process"); !b || err != nil { + if b, err := ev.Fields.HasKey("process"); !b || err != nil { return nil, fmt.Errorf("no process field in event") } - m, ok := tryToMapStr(result.Fields["process"]) + m, ok := tryToMapStr(ev.Fields["process"]) if !ok { return nil, fmt.Errorf("process field type not supported") } + result := ev.Clone() err = mapstr.MergeFieldsDeep(m, processMap, true) if err != nil { return nil, fmt.Errorf("merging enriched fields with event: %w", err) } + result.Fields["process"] = m if p.config.ReplaceFields { if err := p.replaceFields(result); err != nil { @@ -173,7 +173,7 @@ func pidToUInt32(value interface{}) (pid uint32, err error) { // The current version of session view in Kibana expects different values than what are used by auditbeat // for some fields. This function converts these field to have values that will work with session view. // -// This function is temporary, and can be removed when Kibana is updated to work with the auditbeat field values. +// This function is temporary, and can be removed when this Kibana issue is completed: https://github.com/elastic/kibana/issues/179396. func (p *addSessionMetadata) replaceFields(ev *beat.Event) error { kind, err := ev.Fields.GetValue("event.kind") if err != nil { diff --git a/x-pack/auditbeat/processors/sessionmd/timeutils/time.go b/x-pack/auditbeat/processors/sessionmd/timeutils/time.go index 15c62be90084..232074b4b028 100644 --- a/x-pack/auditbeat/processors/sessionmd/timeutils/time.go +++ b/x-pack/auditbeat/processors/sessionmd/timeutils/time.go @@ -16,26 +16,10 @@ import ( ) var ( - bootTime time.Time - ticksPerSecond uint64 - initError error - once sync.Once + getBootTimeOnce = sync.OnceValues(getBootTime) + getTicksPerSecondOnce = sync.OnceValues(getTicksPerSecond) ) -func initialize() { - var err error - bootTime, err = getBootTime() - if err != nil { - initError = err - return - } - - ticksPerSecond, err = getTicksPerSecond() - if err != nil { - initError = err - } -} - func getBootTime() (time.Time, error) { fs, err := procfs.NewDefaultFS() if err != nil { @@ -58,17 +42,17 @@ func getTicksPerSecond() (uint64, error) { } func TicksToNs(ticks uint64) uint64 { - once.Do(initialize) - if initError != nil { + ticksPerSecond, err := getTicksPerSecondOnce() + if err != nil { return 0 } return ticks * uint64(time.Second.Nanoseconds()) / ticksPerSecond } func TimeFromNsSinceBoot(t time.Duration) *time.Time { - once.Do(initialize) - if initError != nil { - return &time.Time{} + bootTime, err := getBootTime() + if err != nil { + return nil } timestamp := bootTime.Add(t) return ×tamp @@ -85,8 +69,8 @@ func TimeFromNsSinceBoot(t time.Duration) *time.Time { // - We store timestamps as nanoseconds, but reduce the precision to 1/100th // second func ReduceTimestampPrecision(timeNs uint64) time.Duration { - once.Do(initialize) - if initError != nil { + ticksPerSecond, err := getTicksPerSecondOnce() + if err != nil { return 0 } return time.Duration(timeNs).Truncate(time.Second / time.Duration(ticksPerSecond)) From 7382b766dddf4f3ef410c977ab753914330e7191 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Thu, 28 Mar 2024 18:09:33 -0700 Subject: [PATCH 32/32] Fix lint error --- x-pack/auditbeat/processors/sessionmd/timeutils/time.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/auditbeat/processors/sessionmd/timeutils/time.go b/x-pack/auditbeat/processors/sessionmd/timeutils/time.go index 232074b4b028..5c8dd7450df0 100644 --- a/x-pack/auditbeat/processors/sessionmd/timeutils/time.go +++ b/x-pack/auditbeat/processors/sessionmd/timeutils/time.go @@ -50,7 +50,7 @@ func TicksToNs(ticks uint64) uint64 { } func TimeFromNsSinceBoot(t time.Duration) *time.Time { - bootTime, err := getBootTime() + bootTime, err := getBootTimeOnce() if err != nil { return nil }