From 9ad246876cce451fb1422d8e3656abdb87f9ae37 Mon Sep 17 00:00:00 2001 From: Scott Gress Date: Thu, 9 Jan 2025 10:00:22 -0600 Subject: [PATCH] Add YARA queries to osquery-perf (#25272) # Overview This PR adds support for remote YARA queries to osquery-perf, so that remote YARA queries can be load-tested. # Details The existing `runLiveQuery()` is updated to branch off into different query running functions based on the content of the query. If the query contains `from yara` and `sigurl`, then the new `runLiveYaraQuery()` function is run which makes a request to the Fleet "get yara rules" API before returning an appropriate response. Otherwise, the new `RunLiveMockQuery()` function is run which includes the previous logic for sending a mock response. # Testing I don't see any automated testing for osquery-perf, but I manually tested in the following way: 1. Started osquery-perf with `go run agent.go` 2. Ran a live query on the new host using ``` SELECT * FROM yara where sigurl="https://localhost:8080/api/osquery/yara/rule1.yar" ``` and verified that the result was as-expected: image I also used a log in Fleet to verify that the "get yara rules" API was really being called. 3. Ran another live query on the host using: ``` SELECT * FROM system_info" ``` and verified that the result was as expected: image I also tested that sending a `sigurl` with the wrong host returns a `live yara query failed because sigurl host did not match server address` error # Checklist for submitter - [X] Added support on fleet's osquery simulator `cmd/osquery-perf` for new osquery data ingestion features. --- cmd/osquery-perf/agent.go | 82 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 81 insertions(+), 1 deletion(-) diff --git a/cmd/osquery-perf/agent.go b/cmd/osquery-perf/agent.go index 0671766fc2ed..388f08537a83 100644 --- a/cmd/osquery-perf/agent.go +++ b/cmd/osquery-perf/agent.go @@ -18,6 +18,7 @@ import ( "net/http" _ "net/http/pprof" "os" + "regexp" "sort" "strconv" "strings" @@ -2090,10 +2091,89 @@ func (a *agent) runLiveQuery(query string) (results []map[string]string, status ss := fleet.OsqueryStatus(1) return []map[string]string{}, &ss, ptr.String("live query failed with error foobar"), nil } - ss := fleet.OsqueryStatus(0) if a.liveQueryNoResultsProb > 0.0 && rand.Float64() <= a.liveQueryNoResultsProb { + ss := fleet.OsqueryStatus(0) return []map[string]string{}, &ss, nil, nil } + + // Switch based on contents of the query. + lcQuery := strings.ToLower(query) + switch { + case strings.Contains(lcQuery, "from yara") && strings.Contains(lcQuery, "sigurl"): + return a.runLiveYaraQuery(query) + default: + return a.runLiveMockQuery(query) + } +} + +func (a *agent) runLiveYaraQuery(query string) (results []map[string]string, status *fleet.OsqueryStatus, message *string, stats *fleet.Stats) { + // Get the URL of the YARA rule to request (i.e. the sigurl). + urlRegex := regexp.MustCompile(`sigurl=(["'])([^"']*)["']`) + matches := urlRegex.FindStringSubmatch(query) + var url string + if len(matches) > 2 { + url = matches[2] + } else { + ss := fleet.OsqueryStatus(1) + return []map[string]string{}, &ss, ptr.String("live yara query failed because a valid sigurl could not be found"), nil + } + + // Osquery validates that the sigurl is one of a configured set, so that it's not + // sending requests to just anywhere. We'll check that it's at least the same host + // as the Fleet server. + if !strings.HasPrefix(url, a.serverAddress) { + ss := fleet.OsqueryStatus(1) + return []map[string]string{}, &ss, ptr.String("live yara query failed because sigurl host did not match server address"), nil + } + + // Make the request. + body := []byte(`{"node_key": "` + a.nodeKey + `"}`) + request, err := http.NewRequest("POST", url, bytes.NewReader(body)) + if err != nil { + ss := fleet.OsqueryStatus(1) + return []map[string]string{}, &ss, ptr.String("live yara query failed due to error creating request"), nil + } + request.Header.Add("Content-type", "application/json") + + // Make the request. + response, err := http.DefaultClient.Do(request) + if err != nil { + ss := fleet.OsqueryStatus(1) + return []map[string]string{}, &ss, ptr.String(fmt.Sprintf("yara request failed to run: %v", err)), nil + } + defer response.Body.Close() + + // For load testing purposes we don't actually care about the response, but check that we at least got one. + if _, err := io.Copy(io.Discard, response.Body); err != nil { + ss := fleet.OsqueryStatus(1) + return []map[string]string{}, &ss, ptr.String(fmt.Sprintf("error reading response from yara API: %v", err)), nil + } + + // Return a response indicating that the file is clean. + ss := fleet.OsqueryStatus(0) + return []map[string]string{ + { + "count": "0", + "matches": "", + "strings": "", + "tags": "", + "sig_group": "", + "sigfile": "", + "sigrule": "", + "sigurl": url, + // Could pull this from the query, but not necessary for load testing. + "path": "/some/path", + }, + }, &ss, nil, &fleet.Stats{ + WallTimeMs: uint64(rand.Intn(1000) * 1000), + UserTime: uint64(rand.Intn(1000)), + SystemTime: uint64(rand.Intn(1000)), + Memory: uint64(rand.Intn(1000)), + } +} + +func (a *agent) runLiveMockQuery(query string) (results []map[string]string, status *fleet.OsqueryStatus, message *string, stats *fleet.Stats) { + ss := fleet.OsqueryStatus(0) return []map[string]string{ { "admindir": "/var/lib/dpkg",