-
Notifications
You must be signed in to change notification settings - Fork 103
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add parser for CarbonBlack repcli output (#1318)
Co-authored-by: seph <seph@kolide.co> Co-authored-by: James Pickett <James-Pickett@users.noreply.github.com>
- Loading branch information
3 people
authored
Sep 8, 2023
1 parent
81def1b
commit d099cfa
Showing
7 changed files
with
480 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,164 @@ | ||
| package repcli | ||
|
|
||
| // repcli is responsible for parsing the output of the CarbonBlack | ||
| // repcli sensor status utility. Some of the output format has | ||
| // changed from the published documentation, as noted here: | ||
| // https://community.carbonblack.com/t5/Knowledge-Base/Endpoint-Standard-How-to-Verify-Sensor-Status-With-RepCLI/ta-p/62524 | ||
| // | ||
| // As a general note, there are a few nuances to this output format that make a fully | ||
| // recursive solution difficult to accomplish cleanly. | ||
| // - a key-value line may be nested 3+ times, but be immediately followed by a top level key | ||
| // - some keys may be duplicated within a given section. these values should be represented as an []string | ||
|
|
||
| import ( | ||
| "bufio" | ||
| "fmt" | ||
| "io" | ||
| "strings" | ||
| "unicode" | ||
| ) | ||
|
|
||
| type ( | ||
| resultMap map[string]any | ||
|
|
||
| repcliLine struct { | ||
| isSectionHeader bool | ||
| indentLevel int | ||
| key string | ||
| value string | ||
| } | ||
| ) | ||
|
|
||
| // formatKey prepares raw (potentially multi-word) key values by: | ||
| // - stripping all surrounding whitespace | ||
| // - coercing the entire string to lowercase | ||
| // - splitting multiple words and joining them as snake_case | ||
| func formatKey(key string) string { | ||
| processed := strings.TrimSpace(strings.ToLower(key)) | ||
| words := strings.Fields(processed) | ||
| return strings.Join(words, "_") | ||
| } | ||
|
|
||
| // parseLine reads a line of text and attempts to pull out the | ||
| // key, value, and key depth (level of nesting) into a repcliLine struct. | ||
| // an empty key-value pair is returned if the line is malformed | ||
| func parseLine(line string) *repcliLine { | ||
| if len(line) == 0 { | ||
| return nil // blank lines are not expected or meaningful | ||
| } | ||
|
|
||
| kv := strings.SplitN(line, ":", 2) | ||
| if len(kv) < 2 { | ||
| return nil // lines without a colon are not expected or meaningful | ||
| } | ||
|
|
||
| indentLen := len(kv[0]) - len(strings.TrimLeftFunc(kv[0], unicode.IsSpace)) | ||
| formattedValue := strings.TrimSpace(kv[1]) | ||
|
|
||
| return &repcliLine{ | ||
| isSectionHeader: (len(formattedValue) == 0), | ||
| indentLevel: indentLen, | ||
| key: formatKey(kv[0]), | ||
| value: formattedValue, | ||
| } | ||
| } | ||
|
|
||
| // updatedKeyPaths takes a running array of lines traversed to get to the latest line (newSection). | ||
| // it does so by iterating over currentPaths to determine the correct placement of newSection based on the | ||
| // indent level for each existing section | ||
| func updatedKeyPaths(currentPaths []*repcliLine, newSection *repcliLine) []*repcliLine { | ||
| updatedPaths := make([]*repcliLine, 0) | ||
|
|
||
| if len(currentPaths) == 0 { | ||
| return append(updatedPaths, newSection) | ||
| } | ||
|
|
||
| for idx, sectionLine := range currentPaths { | ||
| // we only let this fall through if we should add in the new section at the very end | ||
| if newSection.indentLevel > sectionLine.indentLevel { | ||
| updatedPaths = append(updatedPaths, sectionLine) | ||
| continue | ||
| } | ||
|
|
||
| // we've gone too far and need to replace the previous key | ||
| if newSection.indentLevel < sectionLine.indentLevel { | ||
| return append(currentPaths[:idx-1], newSection) | ||
| } | ||
|
|
||
| // this key is at the same level as our new section, replace that in the currentPaths | ||
| return append(currentPaths[:idx], newSection) | ||
| } | ||
|
|
||
| return append(updatedPaths, newSection) | ||
| } | ||
|
|
||
| // setNestedValue works to recursively dive into the resultMap while traversing the | ||
| // lines provided to set the final (deepest) value. | ||
| func setNestedValue(results resultMap, lines []*repcliLine) resultMap { | ||
| if len(lines) == 0 { | ||
| return results | ||
| } | ||
|
|
||
| key, value := lines[0].key, lines[0].value | ||
| if len(lines) == 1 { | ||
| // handle any cases where there is already a value set for key | ||
| switch knownValue := results[key].(type) { | ||
| case []string: | ||
| results[key] = append(knownValue, value) | ||
| case string: | ||
| results[key] = []string{knownValue, value} | ||
| case resultMap, interface{}, nil: | ||
| results[key] = value | ||
| default: | ||
| // if additional nested types are required they should be added above | ||
| results[key] = fmt.Sprintf("unknown type %T requested on value %v", knownValue, value) | ||
| } | ||
|
|
||
| return results | ||
| } | ||
|
|
||
| if _, ok := results[key]; !ok { | ||
| results[key] = make(resultMap, 0) | ||
| } | ||
|
|
||
| results[key] = setNestedValue(results[key].(resultMap), lines[1:]) | ||
|
|
||
| return results | ||
| } | ||
|
|
||
| // repcliParse will take a reader containing stdout data from a cli invocation of repcli. | ||
| // The general approach here is as follows: | ||
| // - read in each line of output, breaking it down into key, optional value, and indentation length | ||
| // - update the paths taken to get to this line (see updatedKeyPaths) | ||
| // - if there is a value to set, set it in results using the currentKeyPaths accumulated (see setNestedValue) | ||
| // | ||
| // We are expecting to parse something like the following into an arbitrarily-nested map[string]any: | ||
| // General Info: | ||
| // | ||
| // Sensor Version: 2.14.0.1234321 | ||
| // DeviceHash: test6b7v9Xo5bX50okW5KABCD+wHxb/YZeSzrZACKo0= | ||
| // | ||
| // Sensor Status: | ||
| // | ||
| // State: Enabled | ||
| func repcliParse(reader io.Reader) (any, error) { | ||
| scanner := bufio.NewScanner(reader) | ||
| results := make(map[string]any) | ||
| currentKeyPaths := make([]*repcliLine, 0) | ||
| for scanner.Scan() { | ||
| line := parseLine(scanner.Text()) | ||
| if line == nil { | ||
| continue | ||
| } | ||
|
|
||
| currentKeyPaths = updatedKeyPaths(currentKeyPaths, line) | ||
|
|
||
| if line.isSectionHeader { | ||
| continue | ||
| } | ||
|
|
||
| results = setNestedValue(results, currentKeyPaths) | ||
| } | ||
|
|
||
| return results, nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,200 @@ | ||
| package repcli | ||
|
|
||
| import ( | ||
| "bytes" | ||
| _ "embed" | ||
| "testing" | ||
|
|
||
| "github.com/stretchr/testify/require" | ||
| ) | ||
|
|
||
| //go:embed test-data/repcli_linux.txt | ||
| var repcli_linux_status []byte | ||
|
|
||
| //go:embed test-data/repcli_darwin.txt | ||
| var repcli_mac_status []byte | ||
|
|
||
| func TestParse(t *testing.T) { | ||
| t.Parallel() | ||
|
|
||
| var tests = []struct { | ||
| name string | ||
| input []byte | ||
| expected map[string]any | ||
| }{ | ||
| { | ||
| name: "empty input", | ||
| expected: resultMap{}, | ||
| }, | ||
| { | ||
| name: "unexpected format input", | ||
| input: []byte(` | ||
| Test: topLevelValue | ||
| Nested: | ||
| Sub: Section | ||
| Double Sub: | ||
| second Level: value | ||
| Triple Nested Flag Test: | ||
| Deepest Flag: deepflag1 | ||
| Deepest Flag: deepflag2 | ||
| Deepest Lone Value: lone Value | ||
| You Should Not See this erroneous L1n3 | ||
| `), | ||
| expected: resultMap{ | ||
| "nested": resultMap{ | ||
| "sub": "Section", | ||
| "double_sub": resultMap{ | ||
| "second_level": "value", | ||
| "triple_nested_flag_test": resultMap{ | ||
| "deepest_flag": []string{"deepflag1", "deepflag2"}, | ||
| "deepest_lone_value": "lone Value", | ||
| }, | ||
| }, | ||
| }, | ||
| "test": "topLevelValue", | ||
| }, | ||
| }, | ||
| { | ||
| name: "repcli linux status", | ||
| input: repcli_linux_status, | ||
| expected: resultMap{ | ||
| "cloud_status": resultMap{ | ||
| "proxy": "No", | ||
| "registered": "Yes", | ||
| "server_address": "https://dev-prod06.example.com", | ||
| }, | ||
| "general_info": resultMap{ | ||
| "devicehash": "test6b7v9Xo5bX50okW5KABCD+wHxb/YZeSzrZACKo0=", | ||
| "deviceid": "123453928", | ||
| "quarantine": "No", | ||
| "sensor_version": "2.14.0.1234321", | ||
| }, | ||
| "rules_status": resultMap{ | ||
| "policy_name": "LinuxDefaultPolicy", | ||
| "policy_timestamp": "02/20/2023", | ||
| }, | ||
| "sensor_status": resultMap{ | ||
| "details": resultMap{ | ||
| "liveresponse": []string{ | ||
| "NoSession", | ||
| "Enabled", | ||
| "NoKillSwitch", | ||
| }, | ||
| }, | ||
| "state": "Enabled", | ||
| }, | ||
| }, | ||
| }, | ||
| { | ||
| name: "repcli mac status", | ||
| input: repcli_mac_status, | ||
| expected: resultMap{ | ||
| "cloud_status": resultMap{ | ||
| "mdm_device_id": "99999999-4C8C-45A0-B3EA-053672776382", | ||
| "next_check-in": "Now", | ||
| "next_cloud_upgrade": "None", | ||
| "platform_type": "CLIENT_ARM64", | ||
| "private_logging": "Disabled", | ||
| "registered": "Yes", | ||
| "server_address": "https://dev-prod05.example.com", | ||
| }, | ||
| "enforcement_status": resultMap{ | ||
| "execution_blocks": "0", | ||
| "network_restrictions": "0", | ||
| }, | ||
| "full_disk_access_configurations": resultMap{ | ||
| "osquery": "Unknown", | ||
| "repmgr": "Not Configured", | ||
| "system_extension": "Unknown", | ||
| "uninstall_helper": "Unknown", | ||
| "uninstall_ui": "Unknown", | ||
| }, | ||
| "general_info": resultMap{ | ||
| "background_scan": "Complete", | ||
| "fips_mode": "Disabled", | ||
| "kernel_file_filter": "Connected", | ||
| "kernel_type": "System Extension", | ||
| "last_reset": "not set", | ||
| "sensor_restarts": "1911", | ||
| "sensor_version": "3.7.2.81", | ||
| "system_extension": "Running", | ||
| }, | ||
| "proxy_settings": resultMap{ | ||
| "proxy_configured": "No", | ||
| }, | ||
| "queues": resultMap{ | ||
| "livequeries": resultMap{ | ||
| "completed": "0", | ||
| "outstanding": "0", | ||
| "peak": "2", | ||
| }, | ||
| "pscevents_batch_upload": resultMap{ | ||
| "failed": "0", | ||
| "mean_data_rate_(b/s)": "7583", | ||
| "pending": "0", | ||
| "uploaded": "1727", | ||
| }, | ||
| "reputation_expedited": resultMap{ | ||
| "last_completed_id": "50", | ||
| "last_queue_id": "50", | ||
| "max_outstanding": "2", | ||
| "outstanding": "0", | ||
| "total_queued": "50", | ||
| }, | ||
| "reputation_resubmit": resultMap{ | ||
| "max_outstanding": "0", | ||
| "outstanding": "0", | ||
| "total_queued": "0", | ||
| }, | ||
| "reputation_slow": resultMap{ | ||
| "demand": "0", | ||
| "ready": "128", | ||
| "resubmit": "0", | ||
| "stale": "715", | ||
| }, | ||
| }, | ||
| "rules_status": resultMap{ | ||
| "active_policies": resultMap{ | ||
| "dc_allow_external_devices_revision[1]": "Enabled(Manifest)", | ||
| "device_control_reporting_policy_revision[5]": "Enabled(Manifest)", | ||
| "eedr_reporting_revision[18]": "Enabled(Manifest)", | ||
| "sensor_telemetry_reporting_policy_revision[3]": "Enabled(Built-in)", | ||
| }, | ||
| "endpoint_standard_product": "Enabled", | ||
| "enterprise_edr_product": "Enabled", | ||
| "policy_name": "Workstations", | ||
| "policy_timestamp": "08/22/2023 15:19:53", | ||
| }, | ||
| "sensor_state": resultMap{ | ||
| "boot_count": "103", | ||
| "details": resultMap{ | ||
| "fulldiskaccess": "NotEnabled", | ||
| "liveresponse": []string{ | ||
| "NoSession", | ||
| "NoKillSwitch", | ||
| "Enabled", | ||
| }, | ||
| }, | ||
| "first_boot_after_os_upgrade": "No", | ||
| "service_uptime": "155110500 ms", | ||
| "service_waketime": "37860000 ms", | ||
| "state": "Enabled", | ||
| "svcstable": "Yes", | ||
| }, | ||
| }, | ||
| }, | ||
| } | ||
|
|
||
| for _, tt := range tests { | ||
| tt := tt | ||
| t.Run(tt.name, func(t *testing.T) { | ||
| t.Parallel() | ||
|
|
||
| p := New() | ||
| result, err := p.Parse(bytes.NewReader(tt.input)) | ||
| require.NoError(t, err, "unexpected error parsing input") | ||
|
|
||
| require.Equal(t, tt.expected, result) | ||
| }) | ||
| } | ||
| } |
Oops, something went wrong.