From 6bc213102b4d68d1156d8efc1febc7908c1677f8 Mon Sep 17 00:00:00 2001 From: Denis Rechkunov Date: Thu, 12 Oct 2023 15:44:27 +0200 Subject: [PATCH 1/2] Optimize memory allocations in the event processing pipeline Previously, every time a processor ran on an event we made a clone of the entire event for two reasons: 1. this event could have some nested maps that are shared among multiple events. 2. in case a processor fails to make a change it should be able to revert its partial changes. This change added a new `EventEditor` wrapper that is used for collecting pending event changes in processors with an option to `Apply` or `Reset` them. Additionally, this `EventEditor` takes care of the efficient memory management when making changes to an event by cloning only the nested maps that processors access or modify. Most of the processors just put new keys or delete existing keys on the root-level, so most of the time the nested maps in the event remain untouched and it does not require the whole event to be cloned. --- libbeat/beat/event.go | 229 +++--- libbeat/beat/event_editor.go | 415 ++++++++++ libbeat/beat/event_editor_test.go | 1110 ++++++++++++++++++++++++++ libbeat/beat/event_test.go | 142 +--- libbeat/processors/processor_test.go | 62 ++ 5 files changed, 1752 insertions(+), 206 deletions(-) create mode 100644 libbeat/beat/event_editor.go create mode 100644 libbeat/beat/event_editor_test.go diff --git a/libbeat/beat/event.go b/libbeat/beat/event.go index 54fc3e27cc39..6db34702b96d 100644 --- a/libbeat/beat/event.go +++ b/libbeat/beat/event.go @@ -19,6 +19,7 @@ package beat import ( "errors" + "fmt" "strings" "time" @@ -26,12 +27,21 @@ import ( "github.com/elastic/elastic-agent-libs/mapstr" ) +type updateMode bool + +var ( + updateModeOverwrite updateMode = true + updateModeNoOverwrite updateMode = false +) + // FlagField fields used to keep information or errors when events are parsed. const FlagField = "log.flags" const ( timestampFieldKey = "@timestamp" metadataFieldKey = "@metadata" + metadataKeyPrefix = metadataFieldKey + "." + metadataKeyOffset = len(metadataKeyPrefix) ) // Event is the common event format shared by all beats. @@ -47,32 +57,62 @@ type Event struct { } var ( - errNoTimestamp = errors.New("value is no timestamp") - errNoMapStr = errors.New("value is no map[string]interface{} type") + ErrValueNotTimestamp = errors.New("value is not a timestamp") + ErrValueNotMapStr = errors.New("value is not `mapstr.M` or `map[string]interface{}` type") + ErrAlterMetadataKey = fmt.Errorf("deleting/replacing %q key is not supported", metadataFieldKey) + ErrMetadataAccess = fmt.Errorf("accessing %q key directly is not supported, try nested keys", metadataFieldKey) + ErrDeleteTimestamp = fmt.Errorf("deleting %q key is not supported", timestampFieldKey) ) // SetID overwrites the "id" field in the events metadata. // If Meta is nil, a new Meta dictionary is created. func (e *Event) SetID(id string) { - if e.Meta == nil { - e.Meta = mapstr.M{} + _, _ = e.PutValue("@metadata._id", id) +} + +func (e *Event) HasKey(key string) (bool, error) { + if key == timestampFieldKey || key == metadataFieldKey { + return true, nil + } + + if subKey, ok := e.metadataSubKey(key); ok { + if e.Meta == nil { + return false, nil + } + return e.Meta.HasKey(subKey) + } + + if e.Fields == nil { + return false, nil } - e.Meta["_id"] = id + + return e.Fields.HasKey(key) } func (e *Event) GetValue(key string) (interface{}, error) { if key == timestampFieldKey { return e.Timestamp, nil - } else if subKey, ok := metadataKey(key); ok { - if subKey == "" || e.Meta == nil { - return e.Meta, nil + } + if key == metadataFieldKey { + return nil, ErrMetadataAccess + } + + if subKey, ok := e.metadataSubKey(key); ok { + if e.Meta == nil { + return nil, mapstr.ErrKeyNotFound } return e.Meta.GetValue(subKey) } + + if e.Fields == nil { + return nil, mapstr.ErrKeyNotFound + } + return e.Fields.GetValue(key) } // Clone creates an exact copy of the event +// TODO DELETE func (e *Event) Clone() *Event { return &Event{ Timestamp: e.Timestamp, @@ -92,7 +132,7 @@ func (e *Event) Clone() *Event { // `DeepUpdateNoOverwrite` is a version of this function that does not // overwrite existing values. func (e *Event) DeepUpdate(d mapstr.M) { - e.deepUpdate(d, true) + e.deepUpdate(d, updateModeOverwrite) } // DeepUpdateNoOverwrite recursively copies the key-value pairs from `d` to various properties of the event. @@ -103,10 +143,69 @@ func (e *Event) DeepUpdate(d mapstr.M) { // via `DeepUpdateNoOverwrite`. // `DeepUpdate` is a version of this function that overwrites existing values. func (e *Event) DeepUpdateNoOverwrite(d mapstr.M) { - e.deepUpdate(d, false) + e.deepUpdate(d, updateModeNoOverwrite) } -func (e *Event) deepUpdate(d mapstr.M, overwrite bool) { +func (e *Event) PutValue(key string, v interface{}) (interface{}, error) { + if key == timestampFieldKey { + return e.setTimestamp(v) + } + if key == metadataFieldKey { + return nil, ErrAlterMetadataKey + } + + if subKey, ok := e.metadataSubKey(key); ok { + if e.Meta == nil { + e.Meta = mapstr.M{} + } + + return e.Meta.Put(subKey, v) + } + + if e.Fields == nil { + e.Fields = mapstr.M{} + } + + return e.Fields.Put(key, v) +} + +func (e *Event) Delete(key string) error { + if key == timestampFieldKey { + return ErrDeleteTimestamp + } + if key == metadataFieldKey { + return ErrAlterMetadataKey + } + if subKey, ok := e.metadataSubKey(key); ok { + if e.Meta == nil { + return nil + } + return e.Meta.Delete(subKey) + } + + if e.Fields == nil { + return nil + } + return e.Fields.Delete(key) +} + +// SetErrorWithOption sets the event error field with the message when the addErrKey is set to true. +// If you want to include the data and field you can pass them as parameters and will be appended into the +// error as fields with the corresponding name. +func (e *Event) SetErrorWithOption(message string, addErrKey bool, data string, field string) { + if addErrKey { + errorField := mapstr.M{"message": message, "type": "json"} + if data != "" { + errorField["data"] = data + } + if field != "" { + errorField["field"] = field + } + e.Fields["error"] = errorField + } +} + +func (e *Event) deepUpdate(d mapstr.M, mode updateMode) { if len(d) == 0 { return } @@ -115,14 +214,17 @@ func (e *Event) deepUpdate(d mapstr.M, overwrite bool) { // However, we must handle it separately since it's a separate field of the event. timestampValue, timestampExists := d[timestampFieldKey] if timestampExists { - if overwrite { - _ = e.setTimestamp(timestampValue) + if mode == updateModeOverwrite { + _, _ = e.setTimestamp(timestampValue) } // Temporary delete it from the update map, // so we can do `e.Fields.DeepUpdate(d)` or // `e.Fields.DeepUpdateNoOverwrite(d)` later delete(d, timestampFieldKey) + defer func() { + d[timestampFieldKey] = timestampValue + }() } // It's supported to update the metadata using this function. @@ -142,9 +244,10 @@ func (e *Event) deepUpdate(d mapstr.M, overwrite bool) { if e.Meta == nil { e.Meta = mapstr.M{} } - if overwrite { + switch mode { + case updateModeOverwrite: e.Meta.DeepUpdate(metaUpdate) - } else { + case updateModeNoOverwrite: e.Meta.DeepUpdateNoOverwrite(metaUpdate) } } @@ -153,17 +256,10 @@ func (e *Event) deepUpdate(d mapstr.M, overwrite bool) { // so we can do `e.Fields.DeepUpdate(d)` or // `e.Fields.DeepUpdateNoOverwrite(d)` later delete(d, metadataFieldKey) - } - - // At the end we revert all changes we made to the update map - defer func() { - if timestampExists { - d[timestampFieldKey] = timestampValue - } - if metaExists { + defer func() { d[metadataFieldKey] = metaValue - } - }() + }() + } if len(d) == 0 { return @@ -173,90 +269,39 @@ func (e *Event) deepUpdate(d mapstr.M, overwrite bool) { e.Fields = mapstr.M{} } - if overwrite { + switch mode { + case updateModeOverwrite: e.Fields.DeepUpdate(d) - } else { + case updateModeNoOverwrite: e.Fields.DeepUpdateNoOverwrite(d) } } -func (e *Event) setTimestamp(v interface{}) error { +func (e *Event) setTimestamp(v interface{}) (interface{}, error) { + // to satisfy the PutValue interface, this function + // must return the overwritten value + prevValue := e.Timestamp + switch ts := v.(type) { case time.Time: e.Timestamp = ts + return prevValue, nil case common.Time: e.Timestamp = time.Time(ts) + return prevValue, nil default: - return errNoTimestamp - } - - return nil -} - -func (e *Event) PutValue(key string, v interface{}) (interface{}, error) { - if key == timestampFieldKey { - err := e.setTimestamp(v) - return nil, err - } else if subKey, ok := metadataKey(key); ok { - if subKey == "" { - switch meta := v.(type) { - case mapstr.M: - e.Meta = meta - case map[string]interface{}: - e.Meta = meta - default: - return nil, errNoMapStr - } - } else if e.Meta == nil { - e.Meta = mapstr.M{} - } - return e.Meta.Put(subKey, v) - } - - return e.Fields.Put(key, v) -} - -func (e *Event) Delete(key string) error { - if subKey, ok := metadataKey(key); ok { - if subKey == "" { - e.Meta = nil - return nil - } - if e.Meta == nil { - return nil - } - return e.Meta.Delete(subKey) + return nil, ErrValueNotTimestamp } - return e.Fields.Delete(key) } -func metadataKey(key string) (string, bool) { - if !strings.HasPrefix(key, metadataFieldKey) { +func (e *Event) metadataSubKey(key string) (string, bool) { + if !strings.HasPrefix(key, metadataKeyPrefix) { return "", false } - subKey := key[len(metadataFieldKey):] + subKey := key[metadataKeyOffset:] if subKey == "" { - return "", true - } - if subKey[0] == '.' { - return subKey[1:], true - } - return "", false -} - -// SetErrorWithOption sets the event error field with the message when the addErrKey is set to true. -// If you want to include the data and field you can pass them as parameters and will be appended into the -// error as fields with the corresponding name. -func (e *Event) SetErrorWithOption(message string, addErrKey bool, data string, field string) { - if addErrKey { - errorField := mapstr.M{"message": message, "type": "json"} - if data != "" { - errorField["data"] = data - } - if field != "" { - errorField["field"] = field - } - e.Fields["error"] = errorField + return "", false } + return subKey, true } diff --git a/libbeat/beat/event_editor.go b/libbeat/beat/event_editor.go new file mode 100644 index 000000000000..1d8ece2be194 --- /dev/null +++ b/libbeat/beat/event_editor.go @@ -0,0 +1,415 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package beat + +import ( + "errors" + "strings" + + "github.com/elastic/elastic-agent-libs/mapstr" +) + +type checkoutMode bool + +var ( + checkoutModeOnlyMaps checkoutMode = false + checkoutModeIncludeValues checkoutMode = true +) + +// TODO rename to `Event` and use it instead of the actual event struct everywhere +type EventAccessor interface { + // GetValue gets a value from the map. The key can be expressed in dot-notation (e.g. x.y). + // If the key does not exist then `mapstr.ErrKeyNotFound` error is returned. + GetValue(key string) (interface{}, error) + + // PutValue associates the specified value with the specified key. If the event + // previously contained a mapping for the key, the old value is replaced and + // returned. The key can be expressed in dot-notation (e.g. x.y) to put a value + // into a nested map. + // + // If you need insert keys containing dots then you must use bracket notation + // to insert values (e.g. m[key] = value). + PutValue(key string, v interface{}) (interface{}, error) + + // Delete value with the given key. + // The key can be expressed in dot-notation (e.g. x.y) + Delete(key string) error + + // DeepUpdate recursively copies the key-value pairs from `d` to various properties of the event. + // When the key equals `@timestamp` it's set as the `Timestamp` property of the event. + // When the key equals `@metadata` the update is routed into the `Meta` map instead of `Fields` + // The rest of the keys are set to the `Fields` map. + // If the key is present and the value is a map as well, the sub-map will be updated recursively + // via `DeepUpdate`. + // `DeepUpdateNoOverwrite` is a version of this function that does not + // overwrite existing values.p + DeepUpdate(d mapstr.M) + + // DeepUpdateNoOverwrite recursively copies the key-value pairs from `d` to various properties of the event. + // The `@timestamp` update is ignored due to "no overwrite" behavior. + // When the key equals `@metadata` the update is routed into the `Meta` map instead of `Fields`. + // The rest of the keys are set to the `Fields` map. + // If the key is present and the value is a map as well, the sub-map will be updated recursively + // via `DeepUpdateNoOverwrite`. + // `DeepUpdate` is a version of this function that overwrites existing values. + DeepUpdateNoOverwrite(d mapstr.M) +} + +// EventEditor is a wrapper that allows to make changes to the wrapped event +// preserving states of the original nested maps by cloning them on demand. +// +// The first time a nested map gets updated it's cloned and, from that moment on, only the copy is modified. +// Once all the changes are collected, users should call `Apply` to copy pending changes to the original event. +// When the changes get applied the pointers to originally referenced nested maps get replaced with pointers to +// modified copies. +// +// This allows us to: +// * avoid cloning the entire event and be more efficient in memory management +// * collect multiple changes and apply them at once, using a transaction-like mechanism (`Apply`/`Reset` functions). +// +// WARNING: +// Events can contains slices which this editor does not preserve. +// It's on the consumer to copy slices if they need to be changed. +// This editor takes care of maps only, not looking into slices. This means maps in slices are not preserved either. +type EventEditor struct { + original *Event + pending *Event + deletions map[string]struct{} +} + +func NewEventEditor(e *Event) *EventEditor { + if e == nil { + e = &Event{} + } + return &EventEditor{ + original: e, + } +} + +// GetValue implements the `EventAccessor` interface. +func (e *EventEditor) GetValue(key string) (interface{}, error) { + if key == metadataFieldKey { + return nil, ErrMetadataAccess + } + + // handle the deletion marks + rootKey := e.rootKey(key) + if rootKey == key && e.checkDeleted(key) { + return nil, mapstr.ErrKeyNotFound + } + + if e.pending != nil { + // We try to retrieve from the `pending` event first, + // since it should have the most recent data. + // + // To check if the nested map was checked-out before + // we need to get the root-level first + val, err := e.pending.GetValue(rootKey) + if err == nil { + if rootKey == key { + // the value might be the end value we're looking for + return val, nil + } else { + // otherwise, we need to retrieve from the nested map + subKey := key[len(rootKey)+1:] + switch nested := val.(type) { + case mapstr.M: + return nested.GetValue(subKey) + case map[string]interface{}: + return mapstr.M(nested).GetValue(subKey) + default: + return nil, mapstr.ErrKeyNotFound + } + } + } + + if !errors.Is(err, mapstr.ErrKeyNotFound) { + return nil, err + } + } + + value, err := e.original.GetValue(key) + if err != nil { + return nil, err + } + + // if the end value is not a map, we can just return it, + // value types will be copied automatically + switch value.(type) { + case mapstr.M, map[string]interface{}: + default: + return value, nil + } + + // if the key leads to a map value or it's in a nested map, + // we must check it out before returning, so we return a clone + // that the consumer can modify + e.checkout(key, checkoutModeOnlyMaps) + + return e.pending.GetValue(key) +} + +// PutValue implements the `EventAccessor` interface. +func (e *EventEditor) PutValue(key string, v interface{}) (interface{}, error) { + if key == metadataFieldKey { + return nil, ErrAlterMetadataKey + } + // checkout only if the key leads to a nested value + e.checkout(key, checkoutModeOnlyMaps) + e.allocatePending() + return e.pending.PutValue(key, v) +} + +// Delete implements the `EventAccessor` interface. +func (e *EventEditor) Delete(key string) error { + if key == timestampFieldKey { + return ErrDeleteTimestamp + } + if key == metadataFieldKey { + return ErrAlterMetadataKey + } + var deleted bool + has, _ := e.original.HasKey(key) + + if has { + if key == e.rootKey(key) { + // if we're trying to delete a root-level key + // it must be deleted from the `original` event when `Apply` is called + // and from the `pending` event (below) if a value with the same key was put there. + e.markAsDeleted(key) + deleted = true + } else { + // if it's not a root-level key, we checkout this entire root-level map + // and delete from the `pending` event instead. + e.checkout(key, checkoutModeOnlyMaps) + } + } + + if e.pending != nil { + err := e.pending.Delete(key) + deleted = deleted || err == nil + } + + if deleted { + return nil + } else { + return mapstr.ErrKeyNotFound + } +} + +// DeepUpdate implements the `EventAccessor` interface. +func (e *EventEditor) DeepUpdate(d mapstr.M) { e.deepUpdate(d, updateModeOverwrite) } + +// DeepUpdateNoOverwrite implements the `EventAccessor` interface. +func (e *EventEditor) DeepUpdateNoOverwrite(d mapstr.M) { e.deepUpdate(d, updateModeNoOverwrite) } + +// Apply write all the changes to the original event making sure that none of the original +// nested maps are modified but replaced with modified clones. +func (e *EventEditor) Apply() { + if e.pending == nil { + return + } + + defer e.Reset() + + e.original.Timestamp = e.pending.Timestamp + for deletedKey := range e.deletions { + _ = e.original.Delete(deletedKey) + } + + // it's enough to overwrite the root-level because + // of the checkout mechanism used earlier + if len(e.pending.Meta) > 0 { + if e.original.Meta == nil { + e.original.Meta = mapstr.M{} + } + for key := range e.pending.Meta { + e.original.Meta[key] = e.pending.Meta[key] + } + } + if len(e.pending.Fields) > 0 { + if e.original.Fields == nil { + e.original.Fields = mapstr.M{} + } + for key := range e.pending.Fields { + e.original.Fields[key] = e.pending.Fields[key] + } + } +} + +// Reset cleans all the pending changes and starts collecting them again. +// This function does not allocate new memory. +func (e *EventEditor) Reset() { + if e.pending == nil { + return + } + e.pending.Timestamp = e.original.Timestamp + for k := range e.deletions { + delete(e.deletions, k) + } + for k := range e.pending.Meta { + delete(e.pending.Meta, k) + } + for k := range e.pending.Fields { + delete(e.pending.Fields, k) + } +} + +// deepUpdate checks out all the necessary root-level keys before running the deep update. +func (e *EventEditor) deepUpdate(d mapstr.M, mode updateMode) { + if len(d) == 0 { + return + } + cm := checkoutModeOnlyMaps + if mode == updateModeNoOverwrite { + cm = checkoutModeIncludeValues + } + + // checkout necessary keys from the original event + for key := range d { + if key == timestampFieldKey { + continue + } + + // we never checkout the whole metadata, only root-level keys + // which are about to get updated + if key == metadataFieldKey { + metaUpdate := d[metadataFieldKey] + switch m := metaUpdate.(type) { + case mapstr.M: + for innerKey := range m { + e.checkout(metadataKeyPrefix+innerKey, cm) + } + case map[string]interface{}: + for innerKey := range m { + e.checkout(metadataKeyPrefix+innerKey, cm) + } + } + continue + } + + e.checkout(key, cm) + } + + e.allocatePending() + e.pending.deepUpdate(d, mode) +} + +// markAsDeleted marks a key for deletion when `Apply` is called +// if it's a root-level key in the original event. +func (e *EventEditor) markAsDeleted(key string) { + dotIdx := e.dotIdx(key) + // nested keys are not marked since nested maps + // are cloned into the `pending` event and altered there + if dotIdx != -1 { + return + } + if e.deletions == nil { + e.deletions = make(map[string]struct{}) + } + e.deletions[key] = struct{}{} +} + +// checkout clones a nested map of the original event to the event with pending changes. +// +// If the key leads to a value nested in a map, we checkout the root-level nested map which means +// the whole sub-tree is recursively cloned, so it can be safely modified. +func (e *EventEditor) checkout(key string, mode checkoutMode) { + // we're always looking only at the root-level + rootKey := e.rootKey(key) + + if e.pending != nil { + // it might be already checked out + checkedOut, _ := e.pending.HasKey(rootKey) + if checkedOut { + return + } + } + + // if there is nothing to checkout - return + value, err := e.original.GetValue(rootKey) + if err != nil { + return + } + + e.allocatePending() + + // we check out only nested maps, and leave root-level value types in the original map + // unless the special `includeValues` mode engaged (used for DeepUpdateNoOverwrite). + switch typedVal := value.(type) { + case mapstr.M: + _, _ = e.pending.PutValue(rootKey, typedVal.Clone()) + case map[string]interface{}: + _, _ = e.pending.PutValue(rootKey, mapstr.M(typedVal).Clone()) + default: + if mode == checkoutModeIncludeValues { + _, _ = e.pending.PutValue(rootKey, typedVal) + } + } +} + +// dotIdx returns index of the first `.` character or `-1` if there is no `.` character. +// Accounts for the `@metadata` subkeys, since it's stored in a separate map, +// root-level keys will be in the `@metadata.*` namespace. +func (e *EventEditor) dotIdx(key string) int { + // metadata keys are special, since they're stored in a separate map + // we don't want to copy the whole map with all metadata, we want + // to checkout only nested maps one by one for efficiency + if strings.HasPrefix(key, metadataKeyPrefix) { + // we start looking after the `@metadata` prefix + dotIdx := strings.Index(key[metadataKeyOffset:], ".") + // if there is no dot in the subkey, then the second segment + // is considered to be a root-level key + if dotIdx == -1 { + return -1 + } + // otherwise we need to offset the dot index by the prefix we removed + return dotIdx + metadataKeyOffset + } + + return strings.Index(key, ".") +} + +// rootKey reduces the key to its root-level. +func (e *EventEditor) rootKey(key string) string { + dotIdx := e.dotIdx(key) + if dotIdx == -1 { + return key + } else { + return key[:dotIdx] + } +} + +// checkDeleted returns `true` if the key was marked for deletion. +// The key can be expressed in dot-notation (e.g. x.y) and if the root-level prefix on +// the key path is deleted the function returns `true`. +func (e *EventEditor) checkDeleted(key string) bool { + rootKey := e.rootKey(key) + _, deleted := e.deletions[rootKey] + return deleted +} + +// allocatePending makes sure that the `pending` event is allocated for collecting changes. +func (e *EventEditor) allocatePending() { + if e.pending != nil { + return + } + e.pending = &Event{ + Timestamp: e.original.Timestamp, + } +} diff --git a/libbeat/beat/event_editor_test.go b/libbeat/beat/event_editor_test.go new file mode 100644 index 000000000000..05b670f675a9 --- /dev/null +++ b/libbeat/beat/event_editor_test.go @@ -0,0 +1,1110 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package beat + +import ( + "fmt" + "strings" + "testing" + "time" + + "github.com/elastic/elastic-agent-libs/mapstr" + + "github.com/stretchr/testify/require" +) + +func TestEventEditor(t *testing.T) { + metadataNestedNestedMap := mapstr.M{ + "metaLevel2Value": "metavalue3", + } + metadataNestedMap := mapstr.M{ + "metaLevel1Map": metadataNestedNestedMap, + } + + fieldsNestedNestedMap := mapstr.M{ + "fieldsLevel2Value": "fieldsvalue3", + } + fieldsNestedMap := mapstr.M{ + "fieldsLevel1Map": fieldsNestedNestedMap, + } + + metaUntouchedMap := mapstr.M{} + fieldsUntouchedMap := mapstr.M{} + + event := &Event{ + Timestamp: time.Now(), + Meta: mapstr.M{ + "metaLevel0Map": metadataNestedMap, + "metaLevel0Value": "metavalue1", + // this key should never be edited by the tests + // to verify that existing keys remain + "metaLevel0Value2": "untouched", + // this map should never be checked out + "metaUntouchedMap": metaUntouchedMap, + }, + Fields: mapstr.M{ + "fieldsLevel0Map": fieldsNestedMap, + "fieldsLevel0Value": "fieldsvalue1", + // this key should never be edited by the tests + // to verify that existing keys remain + "fieldsLevel0Value2": "untouched", + // this map should never be checked out + "fieldsUntouchedMap": fieldsUntouchedMap, + }, + } + + t.Run("rootKey", func(t *testing.T) { + cases := []struct { + val string + exp string + }{ + { + val: "@metadata.some", + exp: "@metadata.some", + }, + { + val: "@metadata.some.nested", + exp: "@metadata.some", + }, + { + val: "some.nested.key", + exp: "some", + }, + { + val: "key", + exp: "key", + }, + } + + for _, tc := range cases { + e := NewEventEditor(event) + t.Run(tc.val, func(t *testing.T) { + require.Equal(t, tc.exp, e.rootKey(tc.val)) + }) + } + }) + + t.Run("empty", func(t *testing.T) { + t.Run("Apply", func(t *testing.T) { + editor := NewEventEditor(nil) + require.NotPanics(t, func() { + editor.Apply() + }) + }) + + t.Run("Reset", func(t *testing.T) { + editor := NewEventEditor(nil) + require.NotPanics(t, func() { + editor.Reset() + }) + }) + + t.Run("Delete", func(t *testing.T) { + editor := NewEventEditor(nil) + require.NotPanics(t, func() { + err := editor.Delete("@metadata.some") + require.Error(t, err) + require.ErrorIs(t, err, mapstr.ErrKeyNotFound) + err = editor.Delete("some") + require.Error(t, err) + require.ErrorIs(t, err, mapstr.ErrKeyNotFound) + editor.Apply() + }) + }) + + t.Run("GetValue", func(t *testing.T) { + editor := NewEventEditor(nil) + require.NotPanics(t, func() { + _, err := editor.GetValue("@metadata.some") + require.Error(t, err) + require.ErrorIs(t, err, mapstr.ErrKeyNotFound) + _, err = editor.GetValue("some") + require.Error(t, err) + require.ErrorIs(t, err, mapstr.ErrKeyNotFound) + }) + }) + + t.Run("PutValue", func(t *testing.T) { + editor := NewEventEditor(nil) + require.NotPanics(t, func() { + prev, err := editor.PutValue("@metadata.some", "value") + require.NoError(t, err) + require.Nil(t, prev) + prev, err = editor.PutValue("some", "value") + require.NoError(t, err) + require.Nil(t, prev) + editor.Apply() + }) + }) + + t.Run("DeepUpdate", func(t *testing.T) { + editor := NewEventEditor(nil) + require.NotPanics(t, func() { + editor.DeepUpdate(mapstr.M{ + "@metadata": mapstr.M{"key": "value"}, + "key": "value", + }) + editor.Apply() + }) + }) + }) + + t.Run("Get", func(t *testing.T) { + t.Run("@timestamp", func(t *testing.T) { + editor := NewEventEditor(event) + val, err := editor.GetValue("@timestamp") + require.NoError(t, err) + require.Equal(t, event.Timestamp, val) + }) + + t.Run("@metadata", func(t *testing.T) { + t.Run("no acess to @metadata key", func(t *testing.T) { + editor := NewEventEditor(event) + metadata, err := editor.GetValue("@metadata") + require.Nil(t, metadata) + require.Error(t, err) + require.ErrorIs(t, err, ErrMetadataAccess) + }) + + t.Run("non-existing key", func(t *testing.T) { + editor := NewEventEditor(event) + val, err := editor.GetValue("@metadata.none") + require.Nil(t, val) + require.Error(t, err) + require.ErrorIs(t, err, mapstr.ErrKeyNotFound) + }) + + t.Run("gets a value type", func(t *testing.T) { + editor := NewEventEditor(event) + requireMapValues(t, editor, map[string]interface{}{ + "@metadata.metaLevel0Value": "metavalue1", + }) + }) + + t.Run("gets a nested map", func(t *testing.T) { + editor := NewEventEditor(event) + nested, err := editor.GetValue("@metadata.metaLevel0Map") + require.NoError(t, err) + requireClonedMaps(t, metadataNestedMap, nested) + }) + + t.Run("gets a deeper nested map", func(t *testing.T) { + editor := NewEventEditor(event) + + nested, err := editor.GetValue("@metadata.metaLevel0Map.metaLevel1Map") + require.NoError(t, err) + requireClonedMaps(t, metadataNestedNestedMap, nested) + + // the higher level map should be also cloned by accessing the inner map + higher, err := editor.GetValue("@metadata.metaLevel0Map") + require.NoError(t, err) + requireClonedMaps(t, metadataNestedMap, higher) + + // the nested map we got previously should be a part of this cloned higher level map + require.IsType(t, mapstr.M{}, higher) + higherMap := higher.(mapstr.M) + nested2, err := higherMap.GetValue("metaLevel1Map") + require.NoError(t, err) + requireSameMap(t, nested, nested2) + }) + + t.Run("returns the same nested map twice", func(t *testing.T) { + editor := NewEventEditor(event) + nested1, err := editor.GetValue("@metadata.metaLevel0Map.metaLevel1Map") + require.NoError(t, err) + nested2, err := editor.GetValue("@metadata.metaLevel0Map.metaLevel1Map") + require.NoError(t, err) + requireSameMap(t, nested2, nested1) + }) + }) + + t.Run("fields", func(t *testing.T) { + t.Run("non-existing key", func(t *testing.T) { + editor := NewEventEditor(event) + val, err := editor.GetValue("none") + require.Nil(t, val) + require.Error(t, err) + require.ErrorIs(t, err, mapstr.ErrKeyNotFound) + }) + + t.Run("gets a value type", func(t *testing.T) { + editor := NewEventEditor(event) + requireMapValues(t, editor, map[string]interface{}{ + "fieldsLevel0Value": "fieldsvalue1", + }) + }) + + t.Run("gets a nested map", func(t *testing.T) { + editor := NewEventEditor(event) + nested, err := editor.GetValue("fieldsLevel0Map") + require.NoError(t, err) + requireClonedMaps(t, fieldsNestedMap, nested) + }) + + t.Run("gets a deeper nested map", func(t *testing.T) { + editor := NewEventEditor(event) + + nested, err := editor.GetValue("fieldsLevel0Map.fieldsLevel1Map") + require.NoError(t, err) + requireClonedMaps(t, fieldsNestedNestedMap, nested) + + // the higher level map should be also cloned by accessing the inner map + higher, err := editor.GetValue("fieldsLevel0Map") + require.NoError(t, err) + requireClonedMaps(t, fieldsNestedMap, higher) + + // the nested map we got previously should be a part of this higher level map + require.IsType(t, mapstr.M{}, higher) + higherMap := higher.(mapstr.M) + nested2, err := higherMap.GetValue("fieldsLevel1Map") + require.NoError(t, err) + requireSameMap(t, nested, nested2) + }) + + t.Run("returns the same nested map twice", func(t *testing.T) { + editor := NewEventEditor(event) + nested1, err := editor.GetValue("fieldsLevel0Map.fieldsLevel1Map") + require.NoError(t, err) + nested2, err := editor.GetValue("fieldsLevel0Map.fieldsLevel1Map") + require.NoError(t, err) + requireSameMap(t, nested2, nested1) + }) + }) + + }) + + t.Run("Delete", func(t *testing.T) { + t.Run("@timestamp", func(t *testing.T) { + editor := NewEventEditor(event) + err := editor.Delete("@timestamp") + require.Error(t, err) + require.ErrorIs(t, err, ErrDeleteTimestamp) + }) + + t.Run("@metadata", func(t *testing.T) { + t.Run("metadata itself", func(t *testing.T) { + editor := NewEventEditor(event) + err := editor.Delete("@metadata") + require.Error(t, err) + require.ErrorIs(t, err, ErrAlterMetadataKey) + }) + + t.Run("non-existent key", func(t *testing.T) { + editor := NewEventEditor(event) + + err := editor.Delete("@metadata.wrong") + require.Error(t, err) + require.ErrorIs(t, err, mapstr.ErrKeyNotFound) + + err = editor.Delete("@metadata.wrong.key") + require.Error(t, err) + require.ErrorIs(t, err, mapstr.ErrKeyNotFound) + }) + + t.Run("root-level value", func(t *testing.T) { + editor := NewEventEditor(event) + key := "@metadata.metaLevel0Value" + value := "metavalue1" + + val1, err := editor.GetValue(key) + require.NoError(t, err) + require.Equal(t, value, val1) + + err = editor.Delete(key) + require.NoError(t, err) + + val2, err := editor.GetValue(key) + require.Error(t, err) + require.ErrorIs(t, err, mapstr.ErrKeyNotFound) + require.Nil(t, val2) + + // checking if the original event still has it + val3, err := event.GetValue(key) + require.NoError(t, err) + require.Equal(t, value, val3) + }) + + t.Run("root-level map", func(t *testing.T) { + editor := NewEventEditor(event) + key := "@metadata.metaLevel0Map" + + err := editor.Delete(key) + require.NoError(t, err) + + val2, err := editor.GetValue(key) + require.Error(t, err) + require.ErrorIs(t, err, mapstr.ErrKeyNotFound) + require.Nil(t, val2) + + // checking if the original event still has it + val3, err := event.GetValue(key) + require.NoError(t, err) + requireSameMap(t, metadataNestedMap, val3) + }) + + t.Run("nested value", func(t *testing.T) { + editor := NewEventEditor(event) + key := "@metadata.metaLevel0Map.metaLevel1Map.metaLevel2Value" + value := "metavalue3" + + val1, err := editor.GetValue(key) + require.NoError(t, err) + require.Equal(t, value, val1) + + err = editor.Delete(key) + require.NoError(t, err) + + val2, err := editor.GetValue(key) + require.Error(t, err) + require.ErrorIs(t, err, mapstr.ErrKeyNotFound) + require.Nil(t, val2) + + // checking if the original event still has it + val3, err := event.GetValue(key) + require.NoError(t, err) + require.Equal(t, value, val3) + }) + + t.Run("nested map", func(t *testing.T) { + editor := NewEventEditor(event) + key := "@metadata.metaLevel0Map.metaLevel1Map" + + err := editor.Delete(key) + require.NoError(t, err) + + val1, err := editor.GetValue(key) + require.Error(t, err) + require.ErrorIs(t, err, mapstr.ErrKeyNotFound) + require.Nil(t, val1) + + // checking if the original event still has it + val2, err := event.GetValue(key) + require.NoError(t, err) + requireSameMap(t, metadataNestedNestedMap, val2) + }) + + t.Run("nested map twice", func(t *testing.T) { + editor := NewEventEditor(event) + key := "@metadata.metaLevel0Map.metaLevel1Map" + + err := editor.Delete(key) + require.NoError(t, err) + + err = editor.Delete(key) + require.Error(t, err) + require.ErrorIs(t, err, mapstr.ErrKeyNotFound) + + // checking if the original event still has it + val1, err := event.GetValue(key) + require.NoError(t, err) + requireSameMap(t, metadataNestedNestedMap, val1) + }) + }) + + t.Run("fields", func(t *testing.T) { + t.Run("non-existent key", func(t *testing.T) { + editor := NewEventEditor(event) + + err := editor.Delete("wrong") + require.Error(t, err) + require.ErrorIs(t, err, mapstr.ErrKeyNotFound) + + err = editor.Delete("wrong.key") + require.Error(t, err) + require.ErrorIs(t, err, mapstr.ErrKeyNotFound) + }) + + t.Run("root-level value", func(t *testing.T) { + editor := NewEventEditor(event) + key := "fieldsLevel0Value" + value := "fieldsvalue1" + + val1, err := editor.GetValue(key) + require.NoError(t, err) + require.Equal(t, value, val1) + + err = editor.Delete(key) + require.NoError(t, err) + + val2, err := editor.GetValue(key) + require.Error(t, err) + require.ErrorIs(t, err, mapstr.ErrKeyNotFound) + require.Nil(t, val2) + + // checking if the original event still has it + val3, err := event.GetValue(key) + require.NoError(t, err) + require.Equal(t, value, val3) + }) + + t.Run("root-level map", func(t *testing.T) { + editor := NewEventEditor(event) + key := "fieldsLevel0Map" + + err := editor.Delete(key) + require.NoError(t, err) + + val2, err := editor.GetValue(key) + require.Error(t, err) + require.ErrorIs(t, err, mapstr.ErrKeyNotFound) + require.Nil(t, val2) + + // checking if the original event still has it + val3, err := event.GetValue(key) + require.NoError(t, err) + requireSameMap(t, fieldsNestedMap, val3) + }) + + t.Run("nested value", func(t *testing.T) { + editor := NewEventEditor(event) + key := "fieldsLevel0Map.fieldsLevel1Map.fieldsLevel2Value" + value := "fieldsvalue3" + + val1, err := editor.GetValue(key) + require.NoError(t, err) + require.Equal(t, value, val1) + + err = editor.Delete(key) + require.NoError(t, err) + + val2, err := editor.GetValue(key) + require.Error(t, err) + require.ErrorIs(t, err, mapstr.ErrKeyNotFound) + require.Nil(t, val2) + + // checking if the original event still has it + val3, err := event.GetValue(key) + require.NoError(t, err) + require.Equal(t, value, val3) + }) + + t.Run("nested map", func(t *testing.T) { + editor := NewEventEditor(event) + key := "fieldsLevel0Map.fieldsLevel1Map" + + err := editor.Delete(key) + require.NoError(t, err) + + val1, err := editor.GetValue(key) + require.Error(t, err) + require.ErrorIs(t, err, mapstr.ErrKeyNotFound) + require.Nil(t, val1) + + // checking if the original event still has it + val2, err := event.GetValue(key) + require.NoError(t, err) + requireSameMap(t, fieldsNestedNestedMap, val2) + }) + + t.Run("nested map twice", func(t *testing.T) { + editor := NewEventEditor(event) + key := "fieldsLevel0Map.fieldsLevel1Map" + + err := editor.Delete(key) + require.NoError(t, err) + + err = editor.Delete(key) + require.Error(t, err) + require.ErrorIs(t, err, mapstr.ErrKeyNotFound) + + // checking if the original event still has it + val1, err := event.GetValue(key) + require.NoError(t, err) + requireSameMap(t, fieldsNestedNestedMap, val1) + }) + }) + }) + + t.Run("PutValue", func(t *testing.T) { + t.Run("@timestamp", func(t *testing.T) { + editor := NewEventEditor(event) + newTs := time.Now().Add(time.Hour) + preVal, err := editor.PutValue("@timestamp", newTs) + require.NoError(t, err) + require.Equal(t, event.Timestamp, preVal) + + val, err := editor.GetValue("@timestamp") + require.NoError(t, err) + require.Equal(t, newTs, val) + + // the original event should not have this change + val, err = event.GetValue("@timestamp") + require.NoError(t, err) + require.Equal(t, event.Timestamp, val) + }) + + t.Run("@metadata", func(t *testing.T) { + t.Run("metadata itself", func(t *testing.T) { + editor := NewEventEditor(event) + prevVal, err := editor.PutValue("@metadata", "some") + require.Error(t, err) + require.ErrorIs(t, err, ErrAlterMetadataKey) + require.Nil(t, prevVal) + }) + + t.Run("new root-level value", func(t *testing.T) { + editor := NewEventEditor(event) + key := "@metadata.new" + value := "value" + prevVal, err := editor.PutValue(key, value) + require.NoError(t, err) + require.Nil(t, prevVal) + + val, err := editor.GetValue(key) + require.NoError(t, err) + require.Equal(t, value, val) + + // the original event should not have it + _, err = event.GetValue(key) + require.Error(t, err) + require.ErrorIs(t, err, mapstr.ErrKeyNotFound) + }) + + t.Run("new root-level map", func(t *testing.T) { + editor := NewEventEditor(event) + key := "@metadata.new" + value := mapstr.M{ + "some": "value", + } + prevVal, err := editor.PutValue(key, value) + require.NoError(t, err) + require.Nil(t, prevVal) + + val, err := editor.GetValue(key) + require.NoError(t, err) + require.Equal(t, value, val) + + // the original event should not have it + _, err = event.GetValue(key) + require.Error(t, err) + require.ErrorIs(t, err, mapstr.ErrKeyNotFound) + }) + + t.Run("new nested value", func(t *testing.T) { + editor := NewEventEditor(event) + key := "@metadata.metaLevel0Map.metaLevel1Map.new" + value := "newvalue" + + prevVal, err := editor.PutValue(key, value) + require.NoError(t, err) + require.Nil(t, prevVal) + + val, err := editor.GetValue(key) + require.NoError(t, err) + require.Equal(t, value, val) + + // the original event should not have it + _, err = event.GetValue(key) + require.Error(t, err) + require.ErrorIs(t, err, mapstr.ErrKeyNotFound) + + // the original `metaLevel0Map` should be cloned/checkedout now + val, err = editor.GetValue("@metadata.metaLevel0Map") + require.NoError(t, err) + requireNotSameMap(t, metadataNestedMap, val) + }) + + t.Run("new nested map", func(t *testing.T) { + editor := NewEventEditor(event) + key := "@metadata.metaLevel0Map.metaLevel1Map.new" + value := mapstr.M{ + "some": "value", + } + + prevVal, err := editor.PutValue(key, value) + require.NoError(t, err) + require.Nil(t, prevVal) + + val, err := editor.GetValue(key) + require.NoError(t, err) + requireSameMap(t, value, val) + + // the original event should not have it + _, err = event.GetValue(key) + require.Error(t, err) + require.ErrorIs(t, err, mapstr.ErrKeyNotFound) + + // the original `metaLevel0Map` should be cloned/checkedout now + val, err = editor.GetValue("@metadata.metaLevel0Map") + require.NoError(t, err) + requireNotSameMap(t, metadataNestedMap, val) + }) + + t.Run("replacing nested value", func(t *testing.T) { + editor := NewEventEditor(event) + key := "@metadata.metaLevel0Map.metaLevel1Map.metaLevel2Value" + replacingValue := "metavalue3" + value := "new" + + prevVal, err := editor.PutValue(key, value) + require.NoError(t, err) + require.Equal(t, replacingValue, prevVal) + + val, err := editor.GetValue(key) + require.NoError(t, err) + require.Equal(t, value, val) + + // the original event should have the previous value + val, err = event.GetValue(key) + require.NoError(t, err) + require.Equal(t, replacingValue, val) + + // the original `metaLevel0Map` should be cloned/checkedout now + val, err = editor.GetValue("@metadata.metaLevel0Map") + require.NoError(t, err) + requireNotSameMap(t, metadataNestedMap, val) + }) + }) + + t.Run("fields", func(t *testing.T) { + t.Run("new root-level value", func(t *testing.T) { + editor := NewEventEditor(event) + key := "new" + value := "value" + prevVal, err := editor.PutValue(key, value) + require.NoError(t, err) + require.Nil(t, prevVal) + + val, err := editor.GetValue(key) + require.NoError(t, err) + require.Equal(t, value, val) + + // the original event should not have it + _, err = event.GetValue(key) + require.Error(t, err) + require.ErrorIs(t, err, mapstr.ErrKeyNotFound) + }) + + t.Run("new root-level map", func(t *testing.T) { + editor := NewEventEditor(event) + key := "new" + value := mapstr.M{ + "some": "value", + } + prevVal, err := editor.PutValue(key, value) + require.NoError(t, err) + require.Nil(t, prevVal) + + val, err := editor.GetValue(key) + require.NoError(t, err) + require.Equal(t, value, val) + + // the original event should not have it + _, err = event.GetValue(key) + require.Error(t, err) + require.ErrorIs(t, err, mapstr.ErrKeyNotFound) + }) + + t.Run("new nested value", func(t *testing.T) { + editor := NewEventEditor(event) + key := "fieldsLevel0Map.fieldsLevel1Map.new" + value := "newvalue" + + prevVal, err := editor.PutValue(key, value) + require.NoError(t, err) + require.Nil(t, prevVal) + + val, err := editor.GetValue(key) + require.NoError(t, err) + require.Equal(t, value, val) + + // the original event should not have it + _, err = event.GetValue(key) + require.Error(t, err) + require.ErrorIs(t, err, mapstr.ErrKeyNotFound) + + // the original `fieldsLevel0Map` should be cloned/checkedout now + val, err = editor.GetValue("fieldsLevel0Map") + require.NoError(t, err) + requireNotSameMap(t, metadataNestedMap, val) + }) + + t.Run("new nested map", func(t *testing.T) { + editor := NewEventEditor(event) + key := "fieldsLevel0Map.fieldsLevel1Map.new" + value := mapstr.M{ + "some": "value", + } + + prevVal, err := editor.PutValue(key, value) + require.NoError(t, err) + require.Nil(t, prevVal) + + val, err := editor.GetValue(key) + require.NoError(t, err) + requireSameMap(t, value, val) + + // the original event should not have it + _, err = event.GetValue(key) + require.Error(t, err) + require.ErrorIs(t, err, mapstr.ErrKeyNotFound) + + // the original `metaLevel0Map` should be cloned/checkedout now + val, err = editor.GetValue("fieldsLevel0Map") + require.NoError(t, err) + requireNotSameMap(t, fieldsNestedMap, val) + }) + + t.Run("replacing nested value", func(t *testing.T) { + editor := NewEventEditor(event) + key := "fieldsLevel0Map.fieldsLevel1Map.fieldsLevel2Value" + replacingValue := "fieldsvalue3" + value := "new" + + prevVal, err := editor.PutValue(key, value) + require.NoError(t, err) + require.Equal(t, replacingValue, prevVal) + + val, err := editor.GetValue(key) + require.NoError(t, err) + require.Equal(t, value, val) + + // the original event should have the previous value + val, err = event.GetValue(key) + require.NoError(t, err) + require.Equal(t, replacingValue, val) + + // the original `metaLevel0Map` should be cloned/checkedout now + val, err = editor.GetValue("fieldsLevel0Map") + require.NoError(t, err) + requireNotSameMap(t, fieldsNestedMap, val) + }) + }) + }) + + t.Run("Apply", func(t *testing.T) { + // we're going to make changes, so working with the clone in this test + cloned := event.Clone() + // need later for address comparison + metaNested := cloned.Meta["metaLevel0Map"] + metaUntouched := cloned.Meta["metaUntouchedMap"] + fieldsNested := cloned.Fields["fieldsLevel0Map"] + fieldsUntouched := cloned.Fields["fieldsUntouchedMap"] + + editor := NewEventEditor(cloned) + + // verify that it does nothing without pending changes + editor.Apply() + requireClonedMaps(t, event.Meta, cloned.Meta) + requireClonedMaps(t, event.Fields, cloned.Fields) + + keysToDelete := []string{ + "@metadata.metaLevel0Map.metaLevel1Map.metaLevel2Value", + "@metadata.metaLevel0Value", + "fieldsLevel0Map.fieldsLevel1Map.fieldsLevel2Value", + "fieldsLevel0Value", + } + for _, key := range keysToDelete { + err := editor.Delete(key) + require.NoError(t, err) + } + newTs := time.Now().Add(time.Hour) + keysToPut := map[string]interface{}{ + "@timestamp": newTs, + "@metadata.metaLevel0Map.metaLevel1Map.new1": "newmetavalue1", + "@metadata.metaLevel0Value": "metareplaced1", + "@metadata.new2": "newmetavalue2", + "fieldsLevel0Map.fieldsLevel1Map.new3": "newfieldsvalue1", + "new4": "newfieldsvalue2", + "fieldsLevel0Value": "fieldsreplaced1", + } + for key, val := range keysToPut { + _, err := editor.PutValue(key, val) + require.NoError(t, err) + } + + // making sure that there are no changes yet + require.Equal(t, event.Timestamp, cloned.Timestamp) + requireClonedMaps(t, event.Meta, cloned.Meta) + requireClonedMaps(t, event.Fields, cloned.Fields) + + expEvent := &Event{ + Timestamp: newTs, + Meta: mapstr.M{ + "metaLevel0Map": mapstr.M{ + "metaLevel1Map": mapstr.M{ + "new1": "newmetavalue1", + }, + }, + "metaLevel0Value": "metareplaced1", + "new2": "newmetavalue2", + "metaLevel0Value2": "untouched", + "metaUntouchedMap": metaUntouchedMap, + }, + Fields: mapstr.M{ + "fieldsLevel0Map": mapstr.M{ + "fieldsLevel1Map": mapstr.M{ + "new3": "newfieldsvalue1", + }, + }, + "new4": "newfieldsvalue2", + "fieldsLevel0Value": "fieldsreplaced1", + "fieldsLevel0Value2": "untouched", + "fieldsUntouchedMap": fieldsUntouchedMap, + }, + } + + editor.Apply() + + require.Equal(t, expEvent.Timestamp, cloned.Timestamp) + require.Equal(t, expEvent.Meta.StringToPrint(), cloned.Meta.StringToPrint()) + require.Equal(t, expEvent.Fields.StringToPrint(), cloned.Fields.StringToPrint()) + + // verifying that only the changed nested maps were cloned + requireNotSameMap(t, metaNested, cloned.Meta["metaLevel0Map"]) + requireNotSameMap(t, fieldsNested, cloned.Fields["fieldsLevel0Map"]) + requireSameMap(t, metaUntouched, cloned.Meta["metaUntouchedMap"]) + requireSameMap(t, fieldsUntouched, cloned.Fields["fieldsUntouchedMap"]) + }) + + t.Run("Reset", func(t *testing.T) { + // we might make changes, so working with the cloned event here + cloned := event.Clone() + metaNested := cloned.Meta["metaLevel0Map"] + fieldsNested := cloned.Fields["fieldsLevel0Map"] + editor := NewEventEditor(cloned) + + // verify that `Reset` does nothing without pending changes + editor.Reset() + requireClonedMaps(t, event.Meta, cloned.Meta) + requireClonedMaps(t, event.Fields, cloned.Fields) + + keysToDelete := []string{ + "@metadata.metaLevel0Map.metaLevel1Map.metaLevel2Value", + "@metadata.metaLevel0Value", + "fieldsLevel0Map.fieldsLevel1Map.fieldsLevel2Value", + "fieldsLevel0Value", + } + for _, key := range keysToDelete { + err := editor.Delete(key) + require.NoError(t, err) + } + newTs := time.Now().Add(time.Hour) + keysToPut := map[string]interface{}{ + "@timestamp": newTs, + "@metadata.metaLevel0Map.metaLevel1Map.new1": "newmetavalue1", + "@metadata.metaLevel0Value": "metareplaced1", + "@metadata.new2": "newmetavalue2", + "fieldsLevel0Map.fieldsLevel1Map.new3": "newfieldsvalue1", + "new4": "newfieldsvalue2", + "fieldsLevel0Value": "fieldsreplaced1", + } + for key, val := range keysToPut { + _, err := editor.PutValue(key, val) + require.NoError(t, err) + } + + // making sure that there are no changes yet + require.Equal(t, event.Timestamp, cloned.Timestamp) + requireClonedMaps(t, event.Meta, cloned.Meta) + requireClonedMaps(t, event.Fields, cloned.Fields) + + editor.Reset() + + require.Equal(t, event.Timestamp, cloned.Timestamp) + requireClonedMaps(t, event.Meta, cloned.Meta) + requireClonedMaps(t, event.Fields, cloned.Fields) + + // verifying that the nested maps were not cloned + requireSameMap(t, metaNested, cloned.Meta["metaLevel0Map"]) + requireSameMap(t, fieldsNested, cloned.Fields["fieldsLevel0Map"]) + + // verify that deletions are reset and every value is back + for _, key := range keysToDelete { + _, err := editor.GetValue(key) + require.NoError(t, err) + } + // verify that all the edits are reset + for key, val := range keysToPut { + got, err := editor.GetValue(key) + if strings.Contains(key, "new") { + require.Error(t, err) + require.ErrorIs(t, err, mapstr.ErrKeyNotFound) + continue + } + require.NoError(t, err) + require.NotEqual(t, val, got) + } + }) + + t.Run("DeepUpdate", func(t *testing.T) { + t.Run("empty", func(t *testing.T) { + cloned := event.Clone() + editor := NewEventEditor(cloned) + editor.DeepUpdate(nil) + editor.Apply() + requireClonedMaps(t, event.Meta, cloned.Meta) + requireClonedMaps(t, event.Fields, cloned.Fields) + }) + + newTs := time.Now().Add(time.Hour) + update := map[string]interface{}{ + "@timestamp": newTs, + "@metadata": map[string]interface{}{ + "metaLevel0Map": mapstr.M{ // mix types on purpose, should support both + "metaLevel1Map": map[string]interface{}{ + "new1": "newmetavalue1", + }, + }, + "metaLevel0Value": "metareplaced1", + "new2": "newmetavalue2", + }, + "fieldsLevel0Map": map[string]interface{}{ + "fieldsLevel1Map": mapstr.M{ + "new3": "newfieldsvalue1", + }, + "newmap": map[string]interface{}{ + "new4": "newfieldsvalue2", + }, + }, + "fieldsLevel0Value": "fieldsreplaced1", + } + + t.Run("overwrite", func(t *testing.T) { + cloned := event.Clone() + editor := NewEventEditor(cloned) + editor.DeepUpdate(update) + + expEvent := &Event{ + Timestamp: newTs, + Meta: mapstr.M{ + "metaLevel0Map": mapstr.M{ + "metaLevel1Map": mapstr.M{ + "metaLevel2Value": "metavalue3", + "new1": "newmetavalue1", + }, + }, + "metaLevel0Value": "metareplaced1", + "metaLevel0Value2": "untouched", + "new2": "newmetavalue2", + "metaUntouchedMap": metaUntouchedMap, + }, + Fields: mapstr.M{ + "fieldsLevel0Map": mapstr.M{ + "fieldsLevel1Map": mapstr.M{ + "fieldsLevel2Value": "fieldsvalue3", + "new3": "newfieldsvalue1", + }, + "newmap": map[string]interface{}{ + "new4": "newfieldsvalue2", + }, + }, + "fieldsLevel0Value": "fieldsreplaced1", + "fieldsLevel0Value2": "untouched", + "fieldsUntouchedMap": fieldsUntouchedMap, + }, + } + + // edited nested maps in metadata and fields should be checked out + requireNotSameMap(t, cloned.Meta["metaLevel0Map"], editor.pending.Meta["metaLevel0Map"]) + requireNotSameMap(t, cloned.Fields["fieldsLevel0Map"], editor.pending.Fields["fieldsLevel0Map"]) + require.Nil(t, editor.pending.Meta["metaUntouchedMap"]) + require.Nil(t, editor.pending.Fields["fieldsUntouchedMap"]) + + editor.Apply() + + require.Equal(t, expEvent.Timestamp, cloned.Timestamp) + requireClonedMaps(t, expEvent.Meta, cloned.Meta) + requireClonedMaps(t, expEvent.Fields, cloned.Fields) + }) + + t.Run("no overwrite", func(t *testing.T) { + cloned := event.Clone() + editor := NewEventEditor(cloned) + editor.DeepUpdateNoOverwrite(update) + + expEvent := &Event{ + // should have the original/non-overwritten timestamp value + Timestamp: event.Timestamp, + Meta: mapstr.M{ + "metaLevel0Map": mapstr.M{ + "metaLevel1Map": mapstr.M{ + "metaLevel2Value": "metavalue3", + "new1": "newmetavalue1", + }, + }, + "metaLevel0Value": "metavalue1", + "metaLevel0Value2": "untouched", + "new2": "newmetavalue2", + "metaUntouchedMap": metaUntouchedMap, + }, + Fields: mapstr.M{ + "fieldsLevel0Map": mapstr.M{ + "fieldsLevel1Map": mapstr.M{ + "fieldsLevel2Value": "fieldsvalue3", + "new3": "newfieldsvalue1", + }, + "newmap": map[string]interface{}{ + "new4": "newfieldsvalue2", + }, + }, + "fieldsLevel0Value": "fieldsvalue1", + "fieldsLevel0Value2": "untouched", + "fieldsUntouchedMap": fieldsUntouchedMap, + }, + } + + // only edited nested maps in metadata and fields should be checked out + requireNotSameMap(t, cloned.Meta["metaLevel0Map"], editor.pending.Meta["metaLevel0Map"]) + requireNotSameMap(t, cloned.Fields["fieldsLevel0Map"], editor.pending.Fields["fieldsLevel0Map"]) + require.Nil(t, editor.pending.Meta["metaUntouchedMap"]) + require.Nil(t, editor.pending.Fields["fieldsUntouchedMap"]) + + editor.Apply() + + require.Equal(t, expEvent.Timestamp, cloned.Timestamp) + requireClonedMaps(t, expEvent.Meta, cloned.Meta) + requireClonedMaps(t, expEvent.Fields, cloned.Fields) + }) + }) +} + +func requireClonedMaps(t *testing.T, expected, actual interface{}) { + t.Helper() + requireNotSameMap(t, expected, actual) + require.IsType(t, mapstr.M{}, expected) + require.IsType(t, mapstr.M{}, actual) + + expectedMap := expected.(mapstr.M) + actualMap := actual.(mapstr.M) + + require.Equal(t, expectedMap.StringToPrint(), actualMap.StringToPrint()) +} + +func requireSameMap(t *testing.T, expected, actual interface{}) { + t.Helper() + expectedAddr := fmt.Sprintf("%p", expected) + actualAddr := fmt.Sprintf("%p", actual) + require.Equalf(t, expectedAddr, actualAddr, "should reference the same map: %s != %s", expectedAddr, actualAddr) +} + +func requireNotSameMap(t *testing.T, expected, actual interface{}) { + t.Helper() + expectedAddr := fmt.Sprintf("%p", expected) + actualAddr := fmt.Sprintf("%p", actual) + require.NotEqualf(t, expectedAddr, actualAddr, "should reference different maps: %s == %s", expectedAddr, actualAddr) +} + +func requireMapValues(t *testing.T, e EventAccessor, expected map[string]interface{}) { + t.Helper() + for key := range expected { + val, err := e.GetValue(key) + require.NoError(t, err) + require.Equal(t, expected[key], val) + } +} diff --git a/libbeat/beat/event_test.go b/libbeat/beat/event_test.go index cd165a3c4593..2719567c1acb 100644 --- a/libbeat/beat/event_test.go +++ b/libbeat/beat/event_test.go @@ -23,6 +23,7 @@ import ( "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/elastic/elastic-agent-libs/mapstr" ) @@ -79,33 +80,32 @@ func newEvent(e mapstr.M) *Event { } } -func BenchmarkTestEventPutGetTimestamp(b *testing.B) { +func TestEventPutGetTimestamp(t *testing.T) { evt := newEmptyEvent() ts := time.Now() - evt.PutValue("@timestamp", ts) + prev, err := evt.PutValue("@timestamp", ts) + require.NoError(t, err) + require.Equal(t, time.Time{}, prev, "previous timestamp should be empty") v, err := evt.GetValue("@timestamp") - if err != nil { - b.Fatal(err) - } - - assert.Equal(b, ts, v) - assert.Equal(b, ts, evt.Timestamp) + require.NoError(t, err) + require.Equal(t, ts, v) + require.Equal(t, ts, evt.Timestamp) // The @timestamp is not written into Fields. - assert.Nil(b, evt.Fields["@timestamp"]) + require.Nil(t, evt.Fields["@timestamp"]) } -func BenchmarkTestDeepUpdate(b *testing.B) { +func TestDeepUpdate(t *testing.T) { ts := time.Now() cases := []struct { - name string - event *Event - update mapstr.M - overwrite bool - expected *Event + name string + event *Event + update mapstr.M + mode updateMode + expected *Event }{ { name: "does nothing if no update", @@ -119,7 +119,7 @@ func BenchmarkTestDeepUpdate(b *testing.B) { update: mapstr.M{ timestampFieldKey: ts, }, - overwrite: true, + mode: updateModeOverwrite, expected: &Event{ Timestamp: ts, Fields: mapstr.M{ @@ -135,7 +135,7 @@ func BenchmarkTestDeepUpdate(b *testing.B) { update: mapstr.M{ timestampFieldKey: time.Now().Add(time.Hour), }, - overwrite: false, + mode: updateModeNoOverwrite, expected: &Event{ Timestamp: ts, Fields: mapstr.M{ @@ -175,7 +175,7 @@ func BenchmarkTestDeepUpdate(b *testing.B) { "second": 42, }, }, - overwrite: false, + mode: updateModeNoOverwrite, expected: &Event{ Meta: mapstr.M{ "first": "initial", @@ -199,7 +199,7 @@ func BenchmarkTestDeepUpdate(b *testing.B) { "second": 42, }, }, - overwrite: true, + mode: updateModeOverwrite, expected: &Event{ Meta: mapstr.M{ "first": "new", @@ -221,7 +221,7 @@ func BenchmarkTestDeepUpdate(b *testing.B) { "first": "new", "second": 42, }, - overwrite: false, + mode: updateModeNoOverwrite, expected: &Event{ Fields: mapstr.M{ "first": "initial", @@ -241,7 +241,7 @@ func BenchmarkTestDeepUpdate(b *testing.B) { "first": "new", "second": 42, }, - overwrite: true, + mode: updateModeOverwrite, expected: &Event{ Fields: mapstr.M{ "first": "new", @@ -268,101 +268,15 @@ func BenchmarkTestDeepUpdate(b *testing.B) { } for _, tc := range cases { - b.Run(tc.name, func(b *testing.B) { - tc.event.deepUpdate(tc.update, tc.overwrite) - assert.Equal(b, tc.expected.Timestamp, tc.event.Timestamp) - assert.Equal(b, tc.expected.Fields, tc.event.Fields) - assert.Equal(b, tc.expected.Meta, tc.event.Meta) + t.Run(tc.name, func(t *testing.T) { + tc.event.deepUpdate(tc.update, tc.mode) + assert.Equal(t, tc.expected.Timestamp, tc.event.Timestamp) + assert.Equal(t, tc.expected.Fields, tc.event.Fields) + assert.Equal(t, tc.expected.Meta, tc.event.Meta) }) } } -func BenchmarkTestEventMetadata(b *testing.B) { - const id = "123" - newMeta := func() mapstr.M { return mapstr.M{"_id": id} } - - b.Run("put", func(b *testing.B) { - evt := newEmptyEvent() - meta := newMeta() - - evt.PutValue("@metadata", meta) - - assert.Equal(b, meta, evt.Meta) - assert.Empty(b, evt.Fields) - }) - - b.Run("get", func(b *testing.B) { - evt := newEmptyEvent() - evt.Meta = newMeta() - - meta, err := evt.GetValue("@metadata") - - assert.NoError(b, err) - assert.Equal(b, evt.Meta, meta) - }) - - b.Run("put sub-key", func(b *testing.B) { - evt := newEmptyEvent() - - evt.PutValue("@metadata._id", id) - - assert.Equal(b, newMeta(), evt.Meta) - assert.Empty(b, evt.Fields) - }) - - b.Run("get sub-key", func(b *testing.B) { - evt := newEmptyEvent() - evt.Meta = newMeta() - - v, err := evt.GetValue("@metadata._id") - - assert.NoError(b, err) - assert.Equal(b, id, v) - }) - - b.Run("delete", func(b *testing.B) { - evt := newEmptyEvent() - evt.Meta = newMeta() - - err := evt.Delete("@metadata") - - assert.NoError(b, err) - assert.Nil(b, evt.Meta) - }) - - b.Run("delete sub-key", func(b *testing.B) { - evt := newEmptyEvent() - evt.Meta = newMeta() - - err := evt.Delete("@metadata._id") - - assert.NoError(b, err) - assert.Empty(b, evt.Meta) - }) - - b.Run("setID", func(b *testing.B) { - evt := newEmptyEvent() - - evt.SetID(id) - - assert.Equal(b, newMeta(), evt.Meta) - }) - - b.Run("put non-metadata", func(b *testing.B) { - evt := newEmptyEvent() - - evt.PutValue("@metadataSpecial", id) - - assert.Equal(b, mapstr.M{"@metadataSpecial": id}, evt.Fields) - }) - - b.Run("delete non-metadata", func(b *testing.B) { - evt := newEmptyEvent() - evt.Meta = newMeta() - - err := evt.Delete("@metadataSpecial") - - assert.Error(b, err) - assert.Equal(b, newMeta(), evt.Meta) - }) +func TestEventFieldsAndMetadata(t *testing.T) { + // TODO re-write all these tests using a case list } diff --git a/libbeat/processors/processor_test.go b/libbeat/processors/processor_test.go index 41ed628fbfb9..b9370eb4489a 100644 --- a/libbeat/processors/processor_test.go +++ b/libbeat/processors/processor_test.go @@ -18,13 +18,16 @@ package processors_test import ( + "fmt" "testing" "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/processors" + "github.com/elastic/beats/v7/libbeat/processors/actions" _ "github.com/elastic/beats/v7/libbeat/processors/actions" _ "github.com/elastic/beats/v7/libbeat/processors/add_cloud_metadata" conf "github.com/elastic/elastic-agent-libs/config" @@ -566,3 +569,62 @@ func TestDropMissingFields(t *testing.T) { assert.Equal(t, expectedEvent, processedEvent.Fields) } + +const ( + fieldCount = 10000 + nestingLevel = 3 +) + +func BenchmarkProcessorsRun(b *testing.B) { + processors := processors.NewList(nil) + key1 := "added.field" + proc1 := actions.NewAddFields(mapstr.M{key1: "first"}, true, true) + processors.AddProcessor(proc1) + key2 := "field-0.field-0" + proc2 := actions.NewAddFields(mapstr.M{key2: "second"}, true, true) + processors.AddProcessor(proc2) + + event := &beat.Event{ + Timestamp: time.Now(), + Meta: mapstr.M{}, + Fields: mapstr.M{}, + } + + generateFields(b, event.Meta, 100, 2) + generateFields(b, event.Fields, 100, 2) + + var ( + processed *beat.Event + err error + ) + + b.Run("processors.Run", func(b *testing.B) { + for i := 0; i < b.N; i++ { + processed, err = processors.Run(event) + require.NoError(b, err) + require.NotNil(b, processed) + } + }) + + added, err := processed.GetValue(key1) + require.NoError(b, err) + require.Equal(b, "first", added) + + added, err = processed.GetValue(key2) + require.NoError(b, err) + require.Equal(b, "second", added) +} + +func generateFields(t require.TestingT, m mapstr.M, count, nesting int) { + for i := 0; i < count; i++ { + var err error + if nesting == 0 { + _, err = m.Put(fmt.Sprintf("field-%d", i), fmt.Sprintf("value-%d", i)) + } else { + nested := mapstr.M{} + generateFields(t, nested, count, nesting-1) + _, err = m.Put(fmt.Sprintf("field-%d", i), nested) + } + require.NoError(t, err) + } +} From b236659975259c5d29d4fd4ddbe2f3c0da4f58a9 Mon Sep 17 00:00:00 2001 From: Denis Rechkunov Date: Sun, 15 Oct 2023 17:48:52 +0200 Subject: [PATCH 2/2] Migrate all processors to using the EventEditor --- libbeat/beat/event.go | 70 ++- libbeat/beat/event_editor.go | 514 ++++++++++++++---- libbeat/beat/event_editor_test.go | 258 ++++++++- libbeat/beat/event_test.go | 10 +- libbeat/beat/pipeline.go | 2 +- libbeat/common/jsontransform/expand.go | 6 +- libbeat/common/jsontransform/jsonhelper.go | 65 +-- libbeat/processors/actions/add_fields.go | 27 +- libbeat/processors/actions/add_labels.go | 6 +- .../actions/add_network_direction.go | 18 +- .../actions/add_network_direction_test.go | 16 +- libbeat/processors/actions/add_tags.go | 6 +- libbeat/processors/actions/append.go | 19 +- libbeat/processors/actions/append_test.go | 30 +- libbeat/processors/actions/common_test.go | 7 +- libbeat/processors/actions/copy_fields.go | 15 +- .../processors/actions/copy_fields_test.go | 18 +- .../processors/actions/decode_base64_field.go | 19 +- .../actions/decode_base64_field_test.go | 18 +- .../processors/actions/decode_json_fields.go | 19 +- .../actions/decode_json_fields_test.go | 77 ++- .../actions/decompress_gzip_field.go | 18 +- .../actions/decompress_gzip_field_test.go | 18 +- .../processors/actions/detect_mime_type.go | 8 +- .../actions/detect_mime_type_test.go | 35 +- libbeat/processors/actions/drop_event.go | 5 +- libbeat/processors/actions/drop_fields.go | 38 +- .../processors/actions/drop_fields_test.go | 51 +- libbeat/processors/actions/extract_field.go | 10 +- .../processors/actions/extract_field_test.go | 26 +- libbeat/processors/actions/include_fields.go | 12 +- .../processors/actions/include_fields_test.go | 8 +- libbeat/processors/actions/rename.go | 16 +- libbeat/processors/actions/rename_test.go | 47 +- libbeat/processors/actions/replace.go | 16 +- libbeat/processors/actions/replace_test.go | 39 +- libbeat/processors/actions/truncate_fields.go | 25 +- .../actions/truncate_fields_test.go | 24 +- .../add_cloud_metadata/add_cloud_metadata.go | 12 +- .../provider_alibaba_cloud_test.go | 12 +- .../provider_aws_ec2_test.go | 13 +- .../provider_azure_vm_test.go | 12 +- .../provider_digital_ocean_test.go | 12 +- .../provider_google_gce_test.go | 60 +- .../provider_hetzner_cloud_test.go | 12 +- .../provider_huawei_cloud_test.go | 12 +- .../provider_openstack_nova_test.go | 12 +- .../provider_tencent_cloud_test.go | 12 +- .../add_docker_metadata.go | 23 +- .../add_docker_metadata_test.go | 112 ++-- .../add_host_metadata/add_host_metadata.go | 18 +- .../add_host_metadata_test.go | 130 +++-- libbeat/processors/add_id/add_id.go | 6 +- libbeat/processors/add_id/add_id_test.go | 40 +- libbeat/processors/conditionals.go | 8 +- libbeat/processors/conditionals_test.go | 14 +- libbeat/processors/namespace_test.go | 6 +- libbeat/processors/processor.go | 33 +- libbeat/processors/processor_test.go | 75 +-- libbeat/processors/safe_processor.go | 4 +- libbeat/processors/safe_processor_test.go | 17 +- .../script/javascript/beatevent_v0.go | 35 +- .../script/javascript/beatevent_v0_test.go | 72 ++- .../script/javascript/javascript.go | 17 +- .../javascript/module/processor/chain.go | 4 +- .../processors/script/javascript/session.go | 28 +- .../script/javascript/session_test.go | 23 +- 67 files changed, 1598 insertions(+), 852 deletions(-) diff --git a/libbeat/beat/event.go b/libbeat/beat/event.go index 6db34702b96d..e748c94a8116 100644 --- a/libbeat/beat/event.go +++ b/libbeat/beat/event.go @@ -38,9 +38,11 @@ var ( const FlagField = "log.flags" const ( - timestampFieldKey = "@timestamp" - metadataFieldKey = "@metadata" - metadataKeyPrefix = metadataFieldKey + "." + TimestampFieldKey = "@timestamp" + MetadataFieldKey = "@metadata" + ErrorFieldKey = "error" + TypeFieldKey = "type" + metadataKeyPrefix = MetadataFieldKey + "." metadataKeyOffset = len(metadataKeyPrefix) ) @@ -59,9 +61,9 @@ type Event struct { var ( ErrValueNotTimestamp = errors.New("value is not a timestamp") ErrValueNotMapStr = errors.New("value is not `mapstr.M` or `map[string]interface{}` type") - ErrAlterMetadataKey = fmt.Errorf("deleting/replacing %q key is not supported", metadataFieldKey) - ErrMetadataAccess = fmt.Errorf("accessing %q key directly is not supported, try nested keys", metadataFieldKey) - ErrDeleteTimestamp = fmt.Errorf("deleting %q key is not supported", timestampFieldKey) + ErrAlterMetadataKey = fmt.Errorf("deleting/replacing %q key is not supported", MetadataFieldKey) + ErrMetadataAccess = fmt.Errorf("accessing %q key directly is not supported, try nested keys", MetadataFieldKey) + ErrDeleteTimestamp = fmt.Errorf("deleting %q key is not supported", TimestampFieldKey) ) // SetID overwrites the "id" field in the events metadata. @@ -71,7 +73,7 @@ func (e *Event) SetID(id string) { } func (e *Event) HasKey(key string) (bool, error) { - if key == timestampFieldKey || key == metadataFieldKey { + if key == TimestampFieldKey || key == MetadataFieldKey { return true, nil } @@ -90,10 +92,10 @@ func (e *Event) HasKey(key string) (bool, error) { } func (e *Event) GetValue(key string) (interface{}, error) { - if key == timestampFieldKey { + if key == TimestampFieldKey { return e.Timestamp, nil } - if key == metadataFieldKey { + if key == MetadataFieldKey { return nil, ErrMetadataAccess } @@ -113,15 +115,15 @@ func (e *Event) GetValue(key string) (interface{}, error) { // Clone creates an exact copy of the event // TODO DELETE -func (e *Event) Clone() *Event { - return &Event{ - Timestamp: e.Timestamp, - Meta: e.Meta.Clone(), - Fields: e.Fields.Clone(), - Private: e.Private, - TimeSeries: e.TimeSeries, - } -} +// func (e *Event) Clone() *Event { +// return &Event{ +// Timestamp: e.Timestamp, +// Meta: e.Meta.Clone(), +// Fields: e.Fields.Clone(), +// Private: e.Private, +// TimeSeries: e.TimeSeries, +// } +// } // DeepUpdate recursively copies the key-value pairs from `d` to various properties of the event. // When the key equals `@timestamp` it's set as the `Timestamp` property of the event. @@ -147,10 +149,10 @@ func (e *Event) DeepUpdateNoOverwrite(d mapstr.M) { } func (e *Event) PutValue(key string, v interface{}) (interface{}, error) { - if key == timestampFieldKey { + if key == TimestampFieldKey { return e.setTimestamp(v) } - if key == metadataFieldKey { + if key == MetadataFieldKey { return nil, ErrAlterMetadataKey } @@ -170,10 +172,10 @@ func (e *Event) PutValue(key string, v interface{}) (interface{}, error) { } func (e *Event) Delete(key string) error { - if key == timestampFieldKey { + if key == TimestampFieldKey { return ErrDeleteTimestamp } - if key == metadataFieldKey { + if key == MetadataFieldKey { return ErrAlterMetadataKey } if subKey, ok := e.metadataSubKey(key); ok { @@ -194,7 +196,7 @@ func (e *Event) Delete(key string) error { // error as fields with the corresponding name. func (e *Event) SetErrorWithOption(message string, addErrKey bool, data string, field string) { if addErrKey { - errorField := mapstr.M{"message": message, "type": "json"} + errorField := mapstr.M{"message": message, TypeFieldKey: "json"} if data != "" { errorField["data"] = data } @@ -205,6 +207,16 @@ func (e *Event) SetErrorWithOption(message string, addErrKey bool, data string, } } +// String returns a string representation of the event. +func (e *Event) String() string { + m := mapstr.M{ + TimestampFieldKey: e.Timestamp, + MetadataFieldKey: e.Meta, + } + m.DeepUpdate(e.Fields) + return m.String() +} + func (e *Event) deepUpdate(d mapstr.M, mode updateMode) { if len(d) == 0 { return @@ -212,7 +224,7 @@ func (e *Event) deepUpdate(d mapstr.M, mode updateMode) { // It's supported to update the timestamp using this function. // However, we must handle it separately since it's a separate field of the event. - timestampValue, timestampExists := d[timestampFieldKey] + timestampValue, timestampExists := d[TimestampFieldKey] if timestampExists { if mode == updateModeOverwrite { _, _ = e.setTimestamp(timestampValue) @@ -221,15 +233,15 @@ func (e *Event) deepUpdate(d mapstr.M, mode updateMode) { // Temporary delete it from the update map, // so we can do `e.Fields.DeepUpdate(d)` or // `e.Fields.DeepUpdateNoOverwrite(d)` later - delete(d, timestampFieldKey) + delete(d, TimestampFieldKey) defer func() { - d[timestampFieldKey] = timestampValue + d[TimestampFieldKey] = timestampValue }() } // It's supported to update the metadata using this function. // However, we must handle it separately since it's a separate field of the event. - metaValue, metaExists := d[metadataFieldKey] + metaValue, metaExists := d[MetadataFieldKey] if metaExists { var metaUpdate mapstr.M @@ -255,9 +267,9 @@ func (e *Event) deepUpdate(d mapstr.M, mode updateMode) { // Temporary delete it from the update map, // so we can do `e.Fields.DeepUpdate(d)` or // `e.Fields.DeepUpdateNoOverwrite(d)` later - delete(d, metadataFieldKey) + delete(d, MetadataFieldKey) defer func() { - d[metadataFieldKey] = metaValue + d[MetadataFieldKey] = metaValue }() } diff --git a/libbeat/beat/event_editor.go b/libbeat/beat/event_editor.go index 1d8ece2be194..60a36dd80ede 100644 --- a/libbeat/beat/event_editor.go +++ b/libbeat/beat/event_editor.go @@ -19,6 +19,7 @@ package beat import ( "errors" + "fmt" "strings" "github.com/elastic/elastic-agent-libs/mapstr" @@ -31,50 +32,51 @@ var ( checkoutModeIncludeValues checkoutMode = true ) -// TODO rename to `Event` and use it instead of the actual event struct everywhere -type EventAccessor interface { - // GetValue gets a value from the map. The key can be expressed in dot-notation (e.g. x.y). - // If the key does not exist then `mapstr.ErrKeyNotFound` error is returned. - GetValue(key string) (interface{}, error) - - // PutValue associates the specified value with the specified key. If the event - // previously contained a mapping for the key, the old value is replaced and - // returned. The key can be expressed in dot-notation (e.g. x.y) to put a value - // into a nested map. - // - // If you need insert keys containing dots then you must use bracket notation - // to insert values (e.g. m[key] = value). - PutValue(key string, v interface{}) (interface{}, error) - - // Delete value with the given key. - // The key can be expressed in dot-notation (e.g. x.y) - Delete(key string) error - - // DeepUpdate recursively copies the key-value pairs from `d` to various properties of the event. - // When the key equals `@timestamp` it's set as the `Timestamp` property of the event. - // When the key equals `@metadata` the update is routed into the `Meta` map instead of `Fields` - // The rest of the keys are set to the `Fields` map. - // If the key is present and the value is a map as well, the sub-map will be updated recursively - // via `DeepUpdate`. - // `DeepUpdateNoOverwrite` is a version of this function that does not - // overwrite existing values.p - DeepUpdate(d mapstr.M) - - // DeepUpdateNoOverwrite recursively copies the key-value pairs from `d` to various properties of the event. - // The `@timestamp` update is ignored due to "no overwrite" behavior. - // When the key equals `@metadata` the update is routed into the `Meta` map instead of `Fields`. - // The rest of the keys are set to the `Fields` map. - // If the key is present and the value is a map as well, the sub-map will be updated recursively - // via `DeepUpdateNoOverwrite`. - // `DeepUpdate` is a version of this function that overwrites existing values. - DeepUpdateNoOverwrite(d mapstr.M) +type EventError struct { + Message string + Field string + Data string + Processor string +} + +func (e EventError) Error() string { + var prefix string + if e.Processor != "" { + prefix += fmt.Sprintf("[processor=%s] ", e.Processor) + } + + if e.Field != "" { + prefix += fmt.Sprintf("[field=%q] ", e.Field) + } + + if e.Data != "" { + prefix += fmt.Sprintf("[data=%s] ", e.Data) + } + return prefix + e.Message +} + +func (e EventError) toMap() mapstr.M { + m := mapstr.M{ + "message": e.Message, + } + if e.Data != "" { + m["data"] = e.Data + } + if e.Field != "" { + m["field"] = e.Field + } + if e.Processor != "" { + m["processor"] = e.Processor + } + + return m } // EventEditor is a wrapper that allows to make changes to the wrapped event // preserving states of the original nested maps by cloning them on demand. // // The first time a nested map gets updated it's cloned and, from that moment on, only the copy is modified. -// Once all the changes are collected, users should call `Apply` to copy pending changes to the original event. +// Once all the changes are collected, users can call `Apply` to apply pending changes to the original event. // When the changes get applied the pointers to originally referenced nested maps get replaced with pointers to // modified copies. // @@ -92,6 +94,7 @@ type EventEditor struct { deletions map[string]struct{} } +// NewEventEditor creates a new event editor for the given event. func NewEventEditor(e *Event) *EventEditor { if e == nil { e = &Event{} @@ -101,28 +104,100 @@ func NewEventEditor(e *Event) *EventEditor { } } -// GetValue implements the `EventAccessor` interface. +// Fields returns the current computed state of changes without applying any changes to the original event. +// +// This function is cloning all the fields from the original event. +// Using this function is slow and expensive on memory, try not to use it unless it's absolutely necessary. +func (e *EventEditor) Fields() mapstr.M { + if e.original.Fields == nil { + if e.pending.Fields != nil { + return e.pending.Fields.Clone() + } + return mapstr.M{} + } + fields := e.original.Fields.Clone() + for key := range e.deletions { + _ = fields.Delete(key) + } + if e.pending != nil { + fields.DeepUpdate(e.pending.Fields) + } + return fields +} + +// FlattenKeys returns all flatten keys for the current pending state of the event. +// This includes original undeleted keys and new unapplied keys. +func (e *EventEditor) FlattenKeys() []string { + uniqueKeys := make(map[string]struct{}) + + if e.original.Meta != nil { + for _, key := range *e.original.Meta.FlattenKeys() { + if e.checkDeleted(key) { + continue + } + uniqueKeys[metadataKeyPrefix+key] = struct{}{} + } + } + if e.original.Fields != nil { + for _, key := range *e.original.Fields.FlattenKeys() { + if e.checkDeleted(key) { + continue + } + uniqueKeys[key] = struct{}{} + } + } + + if e.pending != nil { + if e.pending.Meta != nil { + for _, key := range *e.pending.Meta.FlattenKeys() { + uniqueKeys[metadataKeyPrefix+key] = struct{}{} + } + } + if e.pending.Fields != nil { + for _, key := range *e.pending.Fields.FlattenKeys() { + uniqueKeys[key] = struct{}{} + } + } + } + + result := make([]string, 0, len(uniqueKeys)) + for key := range uniqueKeys { + result = append(result, key) + } + return result +} + +// GetValue gets a value from the event. The key can be expressed in dot-notation (e.g. x.y). +// If the key does not exist then `mapstr.ErrKeyNotFound` error is returned. +// +// If the returned value is a nested map this function always returns a copy, +// not the same nested map referenced by the original event. func (e *EventEditor) GetValue(key string) (interface{}, error) { - if key == metadataFieldKey { + if key == "" { + return nil, mapstr.ErrKeyNotFound + } + if key == MetadataFieldKey { return nil, ErrMetadataAccess } - // handle the deletion marks rootKey := e.rootKey(key) - if rootKey == key && e.checkDeleted(key) { - return nil, mapstr.ErrKeyNotFound - } if e.pending != nil { + // if the root key is empty the original event never had this key + if rootKey == "" { + return e.pending.GetValue(key) + } // We try to retrieve from the `pending` event first, // since it should have the most recent data. // // To check if the nested map was checked-out before - // we need to get the root-level first + // we need to get the root-level first. val, err := e.pending.GetValue(rootKey) + // if the key was found, it was either created by PutValue or + // checked out from the original event. if err == nil { if rootKey == key { - // the value might be the end value we're looking for + // the value might be the root-level end value we're looking for return val, nil } else { // otherwise, we need to retrieve from the nested map @@ -143,6 +218,15 @@ func (e *EventEditor) GetValue(key string) (interface{}, error) { } } + // handle the deletion marks but only on the root-level + // the rest should be covered by looking into the pending event + // where we check out all the nested maps. + if rootKey == key && e.checkDeleted(key) { + return nil, mapstr.ErrKeyNotFound + } + + // in case the key was never checked out and it's still + // present only in the original event value, err := e.original.GetValue(key) if err != nil { return nil, err @@ -164,39 +248,58 @@ func (e *EventEditor) GetValue(key string) (interface{}, error) { return e.pending.GetValue(key) } -// PutValue implements the `EventAccessor` interface. +// PutValue associates the specified value with the specified key. If the event +// previously contained a mapping for the key, the old value is replaced and +// returned. The key can be expressed in dot-notation (e.g. x.y) to put a value +// into a nested map. +// +// If the returned previous value is a nested map this function always returns a copy, +// not the same nested map referenced by the original event. +// +// This changed is not applied to the original event until `Apply` is called. func (e *EventEditor) PutValue(key string, v interface{}) (interface{}, error) { - if key == metadataFieldKey { + switch key { + case MetadataFieldKey: return nil, ErrAlterMetadataKey + case TimestampFieldKey: + e.allocatePending() + return e.pending.PutValue(key, v) + default: + e.allocatePending() + // checkout only if the key leads to a nested value + if !e.checkDeleted(key) { + e.checkout(key, checkoutModeIncludeValues) + } + + return e.pending.PutValue(key, v) } - // checkout only if the key leads to a nested value - e.checkout(key, checkoutModeOnlyMaps) - e.allocatePending() - return e.pending.PutValue(key, v) } -// Delete implements the `EventAccessor` interface. +// Delete value with the given key. +// The key can be expressed in dot-notation (e.g. x.y) +// +// This changed is not applied to the original event until `Apply` is called. func (e *EventEditor) Delete(key string) error { - if key == timestampFieldKey { + if key == TimestampFieldKey { return ErrDeleteTimestamp } - if key == metadataFieldKey { + if key == MetadataFieldKey { return ErrAlterMetadataKey } var deleted bool has, _ := e.original.HasKey(key) - if has { - if key == e.rootKey(key) { + rootKey := e.rootKey(key) + if key == rootKey { // if we're trying to delete a root-level key // it must be deleted from the `original` event when `Apply` is called // and from the `pending` event (below) if a value with the same key was put there. - e.markAsDeleted(key) + e.markAsDeleted(rootKey) deleted = true } else { // if it's not a root-level key, we checkout this entire root-level map - // and delete from the `pending` event instead. - e.checkout(key, checkoutModeOnlyMaps) + // and delete from the copy in `pending` event instead. + e.checkout(rootKey, checkoutModeOnlyMaps) } } @@ -212,26 +315,59 @@ func (e *EventEditor) Delete(key string) error { } } -// DeepUpdate implements the `EventAccessor` interface. +// DeleteAll marks all data to be deleted from the event when `Apply` is called. +func (e *EventEditor) DeleteAll() { + // reset all the changes because + // they don't make sense anymore + e.Reset() + // now we mark all root level keys for deletion + // in both `Meta` and `Fields` maps of the original event + if e.original.Meta != nil { + for key := range e.original.Meta { + e.markAsDeleted(metadataKeyPrefix + key) + } + } + if e.original.Fields != nil { + for key := range e.original.Fields { + e.markAsDeleted(key) + } + } +} + +// DeepUpdate recursively copies the key-value pairs from `d` to various properties of the event. +// When the key equals `@timestamp` it's set as the `Timestamp` property of the event. +// When the key equals `@metadata` the update is routed into the `Meta` map instead of `Fields` +// The rest of the keys are set to the `Fields` map. +// If the key is present and the value is a map as well, the sub-map will be updated recursively +// via `DeepUpdate`. +// `DeepUpdateNoOverwrite` is a version of this function that does not +// overwrite existing values. func (e *EventEditor) DeepUpdate(d mapstr.M) { e.deepUpdate(d, updateModeOverwrite) } -// DeepUpdateNoOverwrite implements the `EventAccessor` interface. +// DeepUpdateNoOverwrite recursively copies the key-value pairs from `d` to various properties of the event. +// The `@timestamp` update is ignored due to "no overwrite" behavior. +// When the key equals `@metadata` the update is routed into the `Meta` map instead of `Fields`. +// The rest of the keys are set to the `Fields` map. +// If the key is present and the value is a map as well, the sub-map will be updated recursively +// via `DeepUpdateNoOverwrite`. +// `DeepUpdate` is a version of this function that overwrites existing values. func (e *EventEditor) DeepUpdateNoOverwrite(d mapstr.M) { e.deepUpdate(d, updateModeNoOverwrite) } // Apply write all the changes to the original event making sure that none of the original -// nested maps are modified but replaced with modified clones. +// nested maps are modified but replaced with modified copies. func (e *EventEditor) Apply() { - if e.pending == nil { - return - } - defer e.Reset() - e.original.Timestamp = e.pending.Timestamp for deletedKey := range e.deletions { _ = e.original.Delete(deletedKey) } + if e.pending == nil { + return + } + + e.original.Timestamp = e.pending.Timestamp + // it's enough to overwrite the root-level because // of the checkout mechanism used earlier if len(e.pending.Meta) > 0 { @@ -270,6 +406,107 @@ func (e *EventEditor) Reset() { } } +// AddTags appends a tag to the tags field of the event. If the tags field does not +// exist then it will be created. If the tags field exists and is not a []string +// then an error will be returned. It does not deduplicate the list of tags. +func (e *EventEditor) AddTags(tags ...string) error { + return e.AddTagsWithKey(mapstr.TagsKey, tags...) +} + +// AddTagsWithKey appends a tag to the given field of the event. If the tags field does not +// exist then it will be created. If the tags field exists and is not a []string +// then an error will be returned. It does not deduplicate the list of tags. +func (e *EventEditor) AddTagsWithKey(key string, tags ...string) error { + if len(tags) == 0 { + return nil + } + e.checkout(key, checkoutModeIncludeValues) + e.allocatePending() + if e.pending.Fields == nil { + e.pending.Fields = mapstr.M{} + } + return mapstr.AddTagsWithKey(e.pending.Fields, key, tags) +} + +// AddError appends an error to the event. If the error field does not +// exist then it will be created. +// If the error field exists and another error was already set, it will be converted to a list +// of errors and new errors will be appended there. +// If there is only one error to add, it will be added directly without a list. +func (e *EventEditor) AddError(ee ...EventError) { + if len(ee) == 0 { + return + } + e.checkout(ErrorFieldKey, checkoutModeIncludeValues) + e.allocatePending() + if e.pending.Fields == nil { + e.pending.Fields = mapstr.M{} + } + + var list []mapstr.M + val, err := e.pending.Fields.GetValue(ErrorFieldKey) + // if the value does not exist yet, we initialize the list of errors + // with the size of the current arguments. + if errors.Is(err, mapstr.ErrKeyNotFound) { + list = make([]mapstr.M, 0, len(ee)) + } else if err != nil { + return + } + // if the value already exists, depending on its type + // we need to reformat it or just append to an existing list. + switch typed := val.(type) { + // there was a single error map, should be replaced with a list, + // the existing error will become an item on this list + case mapstr.M: + list = make([]mapstr.M, 0, len(ee)+1) + list = append(list, typed) + // same as the previous case but typed differently + case map[string]interface{}: + list = make([]mapstr.M, 0, len(ee)+1) + list = append(list, mapstr.M(typed)) + // some code can assign an error as a string + // it's not really expected but we should not lose this value + // so, we convert it into a proper error format. + case string: + list = make([]mapstr.M, 0, len(ee)+1) + list = append(list, EventError{Message: typed}.toMap()) + } + + // after the list is prepared and contains already existing errors + // we can start copying the arguments, converting them to maps + for _, err := range ee { + // an error without a message is not a valid error + if err.Message == "" { + continue + } + m := err.toMap() + list = append(list, m) + } + + // if after all manipulations we still have a single error + // we convert it back into a map instead of adding a list + if len(list) == 1 { + e.pending.Fields[ErrorFieldKey] = list[0] + } else { + e.pending.Fields[ErrorFieldKey] = list + } +} + +// String returns the string representation of the current editor's state. +// This function is slow and should be used for debugging purposes only. +func (e *EventEditor) String() string { + deleteList := make([]string, 0, len(e.deletions)) + for key := range e.deletions { + deleteList = append(deleteList, key) + } + m := mapstr.M{ + "original": e.original.String(), + "pending": e.pending.String(), + "deletions": deleteList, + } + return m.String() +} + // deepUpdate checks out all the necessary root-level keys before running the deep update. func (e *EventEditor) deepUpdate(d mapstr.M, mode updateMode) { if len(d) == 0 { @@ -282,14 +519,14 @@ func (e *EventEditor) deepUpdate(d mapstr.M, mode updateMode) { // checkout necessary keys from the original event for key := range d { - if key == timestampFieldKey { + if key == TimestampFieldKey { continue } // we never checkout the whole metadata, only root-level keys // which are about to get updated - if key == metadataFieldKey { - metaUpdate := d[metadataFieldKey] + if key == MetadataFieldKey { + metaUpdate := d[MetadataFieldKey] switch m := metaUpdate.(type) { case mapstr.M: for innerKey := range m { @@ -313,85 +550,140 @@ func (e *EventEditor) deepUpdate(d mapstr.M, mode updateMode) { // markAsDeleted marks a key for deletion when `Apply` is called // if it's a root-level key in the original event. func (e *EventEditor) markAsDeleted(key string) { - dotIdx := e.dotIdx(key) - // nested keys are not marked since nested maps - // are cloned into the `pending` event and altered there - if dotIdx != -1 { + if key == TimestampFieldKey { return } + rootKey := e.rootKey(key) if e.deletions == nil { e.deletions = make(map[string]struct{}) } - e.deletions[key] = struct{}{} + e.deletions[rootKey] = struct{}{} } // checkout clones a nested map of the original event to the event with pending changes. // // If the key leads to a value nested in a map, we checkout the root-level nested map which means -// the whole sub-tree is recursively cloned, so it can be safely modified. +// the whole sub-tree is recursively cloned, so it can be safely modified later. func (e *EventEditor) checkout(key string, mode checkoutMode) { + if key == TimestampFieldKey { + return + } // we're always looking only at the root-level rootKey := e.rootKey(key) + // if the key is not in the original map, the value is empty + if rootKey == "" { + return + } - if e.pending != nil { - // it might be already checked out - checkedOut, _ := e.pending.HasKey(rootKey) - if checkedOut { + e.allocatePending() + + var dstMap, srcMap mapstr.M + if strings.HasPrefix(rootKey, metadataKeyPrefix) { + if e.original.Meta == nil { + return + } + if e.pending.Meta == nil { + e.pending.Meta = mapstr.M{} + } + dstMap = e.pending.Meta + srcMap = e.original.Meta + rootKey = rootKey[metadataKeyOffset:] + } else { + if e.original.Fields == nil { return } + if e.pending.Fields == nil { + e.pending.Fields = mapstr.M{} + } + dstMap = e.pending.Fields + srcMap = e.original.Fields } - // if there is nothing to checkout - return - value, err := e.original.GetValue(rootKey) - if err != nil { + // it might be already checked out + _, checkedOut := dstMap[rootKey] + if checkedOut { return } - e.allocatePending() + // if there is nothing to checkout - return + value, exists := srcMap[rootKey] + if !exists { + return + } // we check out only nested maps, and leave root-level value types in the original map // unless the special `includeValues` mode engaged (used for DeepUpdateNoOverwrite). switch typedVal := value.(type) { case mapstr.M: - _, _ = e.pending.PutValue(rootKey, typedVal.Clone()) + dstMap[rootKey] = typedVal.Clone() case map[string]interface{}: - _, _ = e.pending.PutValue(rootKey, mapstr.M(typedVal).Clone()) + dstMap[rootKey] = mapstr.M(typedVal).Clone() + // TODO slices? default: if mode == checkoutModeIncludeValues { - _, _ = e.pending.PutValue(rootKey, typedVal) + dstMap[rootKey] = typedVal } } } -// dotIdx returns index of the first `.` character or `-1` if there is no `.` character. +// rootKey reduces the key of the original event to its root-level. +// Keys may have `.` character in them on every level and `.` can also mean a nested depth. +// // Accounts for the `@metadata` subkeys, since it's stored in a separate map, // root-level keys will be in the `@metadata.*` namespace. -func (e *EventEditor) dotIdx(key string) int { - // metadata keys are special, since they're stored in a separate map - // we don't want to copy the whole map with all metadata, we want - // to checkout only nested maps one by one for efficiency +// +// Returns empty string if the key does not exist in the original event. +func (e *EventEditor) rootKey(key string) string { + if key == TimestampFieldKey { + return key + } + var ( + prefix string + m mapstr.M + ) + if strings.HasPrefix(key, metadataKeyPrefix) { - // we start looking after the `@metadata` prefix - dotIdx := strings.Index(key[metadataKeyOffset:], ".") - // if there is no dot in the subkey, then the second segment - // is considered to be a root-level key - if dotIdx == -1 { - return -1 - } - // otherwise we need to offset the dot index by the prefix we removed - return dotIdx + metadataKeyOffset + prefix = metadataKeyPrefix + key = key[metadataKeyOffset:] + m = e.original.Meta + } else { + m = e.original.Fields } - return strings.Index(key, ".") -} + // there is no root-level key with this name + if m == nil { + return "" + } -// rootKey reduces the key to its root-level. -func (e *EventEditor) rootKey(key string) string { - dotIdx := e.dotIdx(key) + // this key may be simple and does not contain dots + dotIdx := strings.IndexRune(key, '.') if dotIdx == -1 { - return key - } else { - return key[:dotIdx] + return prefix + key + } + + // fast lane for the whole key + _, exists := m[key] + if exists { + return prefix + key + } + + // otherwise we start with the first segment + // and keep adding the next segment to the key until the resulting value + // exists on the root level. + var rootKey string + for { + rootKey = key[:dotIdx] + _, exists = m[rootKey] + if exists { + return prefix + rootKey + } + dotIdx = strings.IndexRune(key[dotIdx+1:], '.') + // we checked above for the full key and it didn't exist, + // no need to check again, we return since the key is not found + if dotIdx == -1 { + return "" + } + dotIdx += len(rootKey) + 1 } } @@ -404,7 +696,7 @@ func (e *EventEditor) checkDeleted(key string) bool { return deleted } -// allocatePending makes sure that the `pending` event is allocated for collecting changes. +// allocatePending makes sure that the `pending` event is allocated for collecting new changes. func (e *EventEditor) allocatePending() { if e.pending != nil { return diff --git a/libbeat/beat/event_editor_test.go b/libbeat/beat/event_editor_test.go index 05b670f675a9..6af12ab4bff5 100644 --- a/libbeat/beat/event_editor_test.go +++ b/libbeat/beat/event_editor_test.go @@ -49,6 +49,7 @@ func TestEventEditor(t *testing.T) { event := &Event{ Timestamp: time.Now(), Meta: mapstr.M{ + "a.b": "c", "metaLevel0Map": metadataNestedMap, "metaLevel0Value": "metavalue1", // this key should never be edited by the tests @@ -58,6 +59,7 @@ func TestEventEditor(t *testing.T) { "metaUntouchedMap": metaUntouchedMap, }, Fields: mapstr.M{ + "a.b": "c", "fieldsLevel0Map": fieldsNestedMap, "fieldsLevel0Value": "fieldsvalue1", // this key should never be edited by the tests @@ -69,25 +71,56 @@ func TestEventEditor(t *testing.T) { } t.Run("rootKey", func(t *testing.T) { + event := event.Clone() + event.Meta["some.dot.metakey"] = mapstr.M{ + "that.should": mapstr.M{ + "be": "supported", + }, + } + event.Fields["some.dot.key"] = mapstr.M{ + "that.should": mapstr.M{ + "be": "supported", + }, + } cases := []struct { val string exp string }{ { - val: "@metadata.some", - exp: "@metadata.some", + val: "@metadata.a.b", + exp: "@metadata.a.b", + }, + { + val: "@metadata.metaLevel0Value", + exp: "@metadata.metaLevel0Value", + }, + { + val: "@metadata.metaLevel0Map.metaLevel1Map", + exp: "@metadata.metaLevel0Map", + }, + { + val: "@metadata.some.dot.metakey.that.should.be", + exp: "@metadata.some.dot.metakey", }, { - val: "@metadata.some.nested", - exp: "@metadata.some", + val: "a.b", + exp: "a.b", }, { - val: "some.nested.key", - exp: "some", + val: "fieldsLevel0Map.fieldsLevel1Value", + exp: "fieldsLevel0Map", }, { - val: "key", - exp: "key", + val: "fieldsLevel0Map.fieldsLevel1Map.fieldsLevel2Value", + exp: "fieldsLevel0Map", + }, + { + val: "fieldsLevel0Value", + exp: "fieldsLevel0Value", + }, + { + val: "some.dot.key.that.should.be", + exp: "some.dot.key", }, } @@ -196,6 +229,13 @@ func TestEventEditor(t *testing.T) { }) }) + t.Run("gets a root-level dot-key", func(t *testing.T) { + editor := NewEventEditor(event) + requireMapValues(t, editor, map[string]interface{}{ + "@metadata.a.b": "c", + }) + }) + t.Run("gets a nested map", func(t *testing.T) { editor := NewEventEditor(event) nested, err := editor.GetValue("@metadata.metaLevel0Map") @@ -249,6 +289,13 @@ func TestEventEditor(t *testing.T) { }) }) + t.Run("gets a root-level dot-key", func(t *testing.T) { + editor := NewEventEditor(event) + requireMapValues(t, editor, map[string]interface{}{ + "a.b": "c", + }) + }) + t.Run("gets a nested map", func(t *testing.T) { editor := NewEventEditor(event) nested, err := editor.GetValue("fieldsLevel0Map") @@ -339,6 +386,29 @@ func TestEventEditor(t *testing.T) { require.Equal(t, value, val3) }) + t.Run("root-level dot-key", func(t *testing.T) { + editor := NewEventEditor(event) + key := "@metadata.a.b" + value := "c" + + val1, err := editor.GetValue(key) + require.NoError(t, err) + require.Equal(t, value, val1) + + err = editor.Delete(key) + require.NoError(t, err) + + val2, err := editor.GetValue(key) + require.Error(t, err) + require.ErrorIs(t, err, mapstr.ErrKeyNotFound) + require.Nil(t, val2) + + // checking if the original event still has it + val3, err := event.GetValue(key) + require.NoError(t, err) + require.Equal(t, value, val3) + }) + t.Run("root-level map", func(t *testing.T) { editor := NewEventEditor(event) key := "@metadata.metaLevel0Map" @@ -452,6 +522,29 @@ func TestEventEditor(t *testing.T) { require.Equal(t, value, val3) }) + t.Run("root-level dot-key", func(t *testing.T) { + editor := NewEventEditor(event) + key := "a.b" + value := "c" + + val1, err := editor.GetValue(key) + require.NoError(t, err) + require.Equal(t, value, val1) + + err = editor.Delete(key) + require.NoError(t, err) + + val2, err := editor.GetValue(key) + require.Error(t, err) + require.ErrorIs(t, err, mapstr.ErrKeyNotFound) + require.Nil(t, val2) + + // checking if the original event still has it + val3, err := event.GetValue(key) + require.NoError(t, err) + require.Equal(t, value, val3) + }) + t.Run("root-level map", func(t *testing.T) { editor := NewEventEditor(event) key := "fieldsLevel0Map" @@ -595,7 +688,27 @@ func TestEventEditor(t *testing.T) { require.ErrorIs(t, err, mapstr.ErrKeyNotFound) }) - t.Run("new nested value", func(t *testing.T) { + t.Run("existing root-level dot-key", func(t *testing.T) { + editor := NewEventEditor(event) + key := "@metadata.a.b" + value := mapstr.M{ + "some": "value", + } + prevVal, err := editor.PutValue(key, value) + require.NoError(t, err) + require.Equal(t, "c", prevVal) + + val, err := editor.GetValue(key) + require.NoError(t, err) + require.Equal(t, value, val) + + // the original event should have the previous value + val, err = event.GetValue(key) + require.NoError(t, err) + require.Equal(t, "c", val) + }) + + t.Run("new nested value in existing map", func(t *testing.T) { editor := NewEventEditor(event) key := "@metadata.metaLevel0Map.metaLevel1Map.new" value := "newvalue" @@ -619,6 +732,25 @@ func TestEventEditor(t *testing.T) { requireNotSameMap(t, metadataNestedMap, val) }) + t.Run("absolutely new nested value", func(t *testing.T) { + editor := NewEventEditor(event) + key := "@metadata.new1.new2.new3" + value := "newvalue" + + prevVal, err := editor.PutValue(key, value) + require.NoError(t, err) + require.Nil(t, prevVal) + + val, err := editor.GetValue(key) + require.NoError(t, err) + require.Equal(t, value, val) + + // the original event should not have it + _, err = event.GetValue(key) + require.Error(t, err) + require.ErrorIs(t, err, mapstr.ErrKeyNotFound) + }) + t.Run("new nested map", func(t *testing.T) { editor := NewEventEditor(event) key := "@metadata.metaLevel0Map.metaLevel1Map.new" @@ -710,7 +842,27 @@ func TestEventEditor(t *testing.T) { require.ErrorIs(t, err, mapstr.ErrKeyNotFound) }) - t.Run("new nested value", func(t *testing.T) { + t.Run("existing root-level dot-key", func(t *testing.T) { + editor := NewEventEditor(event) + key := "a.b" + value := mapstr.M{ + "some": "value", + } + prevVal, err := editor.PutValue(key, value) + require.NoError(t, err) + require.Equal(t, "c", prevVal) + + val, err := editor.GetValue(key) + require.NoError(t, err) + require.Equal(t, value, val) + + // the original event should have the previous value + val, err = event.GetValue(key) + require.NoError(t, err) + require.Equal(t, "c", val) + }) + + t.Run("new nested value in existing map", func(t *testing.T) { editor := NewEventEditor(event) key := "fieldsLevel0Map.fieldsLevel1Map.new" value := "newvalue" @@ -731,7 +883,26 @@ func TestEventEditor(t *testing.T) { // the original `fieldsLevel0Map` should be cloned/checkedout now val, err = editor.GetValue("fieldsLevel0Map") require.NoError(t, err) - requireNotSameMap(t, metadataNestedMap, val) + requireNotSameMap(t, fieldsNestedMap, val) + }) + + t.Run("absolutely new nested value", func(t *testing.T) { + editor := NewEventEditor(event) + key := "new1.new2.new3" + value := "newvalue" + + prevVal, err := editor.PutValue(key, value) + require.NoError(t, err) + require.Nil(t, prevVal) + + val, err := editor.GetValue(key) + require.NoError(t, err) + require.Equal(t, value, val) + + // the original event should not have it + _, err = event.GetValue(key) + require.Error(t, err) + require.ErrorIs(t, err, mapstr.ErrKeyNotFound) }) t.Run("new nested map", func(t *testing.T) { @@ -785,6 +956,27 @@ func TestEventEditor(t *testing.T) { requireNotSameMap(t, fieldsNestedMap, val) }) }) + + t.Run("type conflict", func(t *testing.T) { + event := &Event{ + Meta: mapstr.M{ + "a": 9, + "c": 10, + }, + Fields: mapstr.M{ + "a": 9, + "c": 10, + }, + } + + editor := NewEventEditor(event) + _, err := editor.PutValue("a.c", 10) + require.Error(t, err) + require.Equal(t, "expected map but type is int", err.Error()) + _, err = editor.PutValue("a.value", 9) + require.Error(t, err) + require.Equal(t, "expected map but type is int", err.Error()) + }) }) t.Run("Apply", func(t *testing.T) { @@ -836,6 +1028,7 @@ func TestEventEditor(t *testing.T) { expEvent := &Event{ Timestamp: newTs, Meta: mapstr.M{ + "a.b": "c", "metaLevel0Map": mapstr.M{ "metaLevel1Map": mapstr.M{ "new1": "newmetavalue1", @@ -847,6 +1040,7 @@ func TestEventEditor(t *testing.T) { "metaUntouchedMap": metaUntouchedMap, }, Fields: mapstr.M{ + "a.b": "c", "fieldsLevel0Map": mapstr.M{ "fieldsLevel1Map": mapstr.M{ "new3": "newfieldsvalue1", @@ -983,6 +1177,7 @@ func TestEventEditor(t *testing.T) { expEvent := &Event{ Timestamp: newTs, Meta: mapstr.M{ + "a.b": "c", "metaLevel0Map": mapstr.M{ "metaLevel1Map": mapstr.M{ "metaLevel2Value": "metavalue3", @@ -995,6 +1190,7 @@ func TestEventEditor(t *testing.T) { "metaUntouchedMap": metaUntouchedMap, }, Fields: mapstr.M{ + "a.b": "c", "fieldsLevel0Map": mapstr.M{ "fieldsLevel1Map": mapstr.M{ "fieldsLevel2Value": "fieldsvalue3", @@ -1032,6 +1228,7 @@ func TestEventEditor(t *testing.T) { // should have the original/non-overwritten timestamp value Timestamp: event.Timestamp, Meta: mapstr.M{ + "a.b": "c", "metaLevel0Map": mapstr.M{ "metaLevel1Map": mapstr.M{ "metaLevel2Value": "metavalue3", @@ -1044,6 +1241,7 @@ func TestEventEditor(t *testing.T) { "metaUntouchedMap": metaUntouchedMap, }, Fields: mapstr.M{ + "a.b": "c", "fieldsLevel0Map": mapstr.M{ "fieldsLevel1Map": mapstr.M{ "fieldsLevel2Value": "fieldsvalue3", @@ -1072,6 +1270,32 @@ func TestEventEditor(t *testing.T) { requireClonedMaps(t, expEvent.Fields, cloned.Fields) }) }) + + t.Run("hierarchy", func(t *testing.T) { + event := &Event{ + Fields: mapstr.M{ + "a.b": 1, + }, + } + editor := NewEventEditor(event) + err := editor.Delete("a.b") + require.NoError(t, err) + + prev, err := editor.PutValue("a.b.c", 1) + require.NoError(t, err) + require.Nil(t, prev) + + expFields := mapstr.M{ + "a": mapstr.M{ + "b": mapstr.M{ + "c": 1, + }, + }, + } + + editor.Apply() + requireClonedMaps(t, expFields, event.Fields) + }) } func requireClonedMaps(t *testing.T, expected, actual interface{}) { @@ -1100,7 +1324,7 @@ func requireNotSameMap(t *testing.T, expected, actual interface{}) { require.NotEqualf(t, expectedAddr, actualAddr, "should reference different maps: %s == %s", expectedAddr, actualAddr) } -func requireMapValues(t *testing.T, e EventAccessor, expected map[string]interface{}) { +func requireMapValues(t *testing.T, e *EventEditor, expected map[string]interface{}) { t.Helper() for key := range expected { val, err := e.GetValue(key) @@ -1108,3 +1332,13 @@ func requireMapValues(t *testing.T, e EventAccessor, expected map[string]interfa require.Equal(t, expected[key], val) } } + +func (e *Event) Clone() *Event { + return &Event{ + Timestamp: e.Timestamp, + Meta: e.Meta.Clone(), + Fields: e.Fields.Clone(), + Private: e.Private, + TimeSeries: e.TimeSeries, + } +} diff --git a/libbeat/beat/event_test.go b/libbeat/beat/event_test.go index 2719567c1acb..0c09eb7345e9 100644 --- a/libbeat/beat/event_test.go +++ b/libbeat/beat/event_test.go @@ -117,7 +117,7 @@ func TestDeepUpdate(t *testing.T) { name: "updates timestamp", event: newEvent(mapstr.M{}), update: mapstr.M{ - timestampFieldKey: ts, + TimestampFieldKey: ts, }, mode: updateModeOverwrite, expected: &Event{ @@ -133,7 +133,7 @@ func TestDeepUpdate(t *testing.T) { "Timestamp": ts, }), update: mapstr.M{ - timestampFieldKey: time.Now().Add(time.Hour), + TimestampFieldKey: time.Now().Add(time.Hour), }, mode: updateModeNoOverwrite, expected: &Event{ @@ -147,7 +147,7 @@ func TestDeepUpdate(t *testing.T) { name: "initializes metadata if nil", event: newEvent(mapstr.M{}), update: mapstr.M{ - metadataFieldKey: mapstr.M{ + MetadataFieldKey: mapstr.M{ "first": "new", "second": 42, }, @@ -170,7 +170,7 @@ func TestDeepUpdate(t *testing.T) { }, }), update: mapstr.M{ - metadataFieldKey: mapstr.M{ + MetadataFieldKey: mapstr.M{ "first": "new", "second": 42, }, @@ -194,7 +194,7 @@ func TestDeepUpdate(t *testing.T) { }, }), update: mapstr.M{ - metadataFieldKey: mapstr.M{ + MetadataFieldKey: mapstr.M{ "first": "new", "second": 42, }, diff --git a/libbeat/beat/pipeline.go b/libbeat/beat/pipeline.go index 8e8b285042c4..bb409a7a9d9f 100644 --- a/libbeat/beat/pipeline.go +++ b/libbeat/beat/pipeline.go @@ -155,7 +155,7 @@ type ProcessorList interface { // registered with the publisher pipeline. type Processor interface { String() string // print full processor description - Run(in *Event) (event *Event, err error) + Run(*EventEditor) (dropped bool, err error) } // PublishMode enum sets some requirements on the client connection to the beats diff --git a/libbeat/common/jsontransform/expand.go b/libbeat/common/jsontransform/expand.go index be07c4200740..3cd6f4db8565 100644 --- a/libbeat/common/jsontransform/expand.go +++ b/libbeat/common/jsontransform/expand.go @@ -31,10 +31,12 @@ import ( // conflicts (i.e. a common prefix where one field is an object and another // is a non-object), an error key is added to the event if add_error_key // is enabled. -func ExpandFields(logger *logp.Logger, event *beat.Event, m mapstr.M, addErrorKey bool) { +func ExpandFields(logger *logp.Logger, event *beat.EventEditor, m mapstr.M, addErrorKey bool) { if err := expandFields(m); err != nil { logger.Errorf("JSON: failed to expand fields: %s", err) - event.SetErrorWithOption(err.Error(), addErrorKey, "", "") + if addErrorKey { + event.AddError(beat.EventError{Message: err.Error()}) + } } } diff --git a/libbeat/common/jsontransform/jsonhelper.go b/libbeat/common/jsontransform/jsonhelper.go index 15d70ae929ac..b08625b709d0 100644 --- a/libbeat/common/jsontransform/jsonhelper.go +++ b/libbeat/common/jsontransform/jsonhelper.go @@ -23,7 +23,6 @@ import ( "time" "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/elastic-agent-libs/mapstr" ) const ( @@ -37,78 +36,68 @@ var ( ) // WriteJSONKeys writes the json keys to the given event based on the overwriteKeys option and the addErrKey -func WriteJSONKeys(event *beat.Event, keys map[string]interface{}, expandKeys, overwriteKeys, addErrKey bool) { +func WriteJSONKeys(event *beat.EventEditor, keys map[string]interface{}, expandKeys, overwriteKeys, addErrKey bool) { + setError := func(string, string) {} + if addErrKey { + setError = func(msg, field string) { + event.AddError(beat.EventError{Message: msg, Field: field}) + } + } if expandKeys { if err := expandFields(keys); err != nil { - event.SetErrorWithOption(err.Error(), addErrKey, "", "") + setError(err.Error(), "") return } } if !overwriteKeys { - // @timestamp and @metadata fields are root-level fields. We remove them so they - // don't become part of event.Fields. - removeKeys(keys, "@timestamp", "@metadata") - // Then, perform deep update without overwriting - event.Fields.DeepUpdateNoOverwrite(keys) + event.DeepUpdateNoOverwrite(keys) return } for k, v := range keys { switch k { - case "@timestamp": + case beat.TimestampFieldKey: vstr, ok := v.(string) if !ok { - event.SetErrorWithOption("@timestamp not overwritten (not string)", addErrKey, "", "") + setError("not overwritten (not a string)", beat.TimestampFieldKey) + removeKeys(keys, k) continue } // @timestamp must be of format RFC3339 or ISO8601 ts, err := parseTimestamp(vstr) if err != nil { - event.SetErrorWithOption(fmt.Sprintf("@timestamp not overwritten (parse error on %s)", vstr), addErrKey, "", "") + setError(fmt.Sprintf("timestamp parse error on %s", vstr), beat.TimestampFieldKey) + removeKeys(keys, k) continue } - event.Timestamp = ts - - case "@metadata": - switch m := v.(type) { - case map[string]string: - if event.Meta == nil && len(m) > 0 { - event.Meta = mapstr.M{} - } - for meta, value := range m { - event.Meta[meta] = value - } - - case map[string]interface{}: - if event.Meta == nil { - event.Meta = mapstr.M{} - } - event.Meta.DeepUpdate(mapstr.M(m)) - + keys[k] = ts + case beat.MetadataFieldKey: + switch v.(type) { + case map[string]string, map[string]interface{}: default: - event.SetErrorWithOption("failed to update @metadata", addErrKey, "", "") + setError("can't replace with a value", beat.MetadataFieldKey) + removeKeys(keys, k) } - case "type": + case beat.TypeFieldKey: vstr, ok := v.(string) if !ok { - event.SetErrorWithOption("type not overwritten (not string)", addErrKey, "", "") + setError("not overwritten (not a string)", beat.TypeFieldKey) + removeKeys(keys, k) continue } if len(vstr) == 0 || vstr[0] == '_' { - event.SetErrorWithOption(fmt.Sprintf("type not overwritten (invalid value [%s])", vstr), addErrKey, "", "") + setError(fmt.Sprintf("not overwritten (invalid value [%s])", vstr), beat.TypeFieldKey) + removeKeys(keys, k) continue } - event.Fields[k] = vstr + keys[k] = vstr } } - // We have accounted for @timestamp, @metadata, type above. So let's remove these keys and - // deep update the event with the rest of the keys. - removeKeys(keys, "@timestamp", "@metadata", "type") - event.Fields.DeepUpdate(keys) + event.DeepUpdate(keys) } func removeKeys(keys map[string]interface{}, names ...string) { diff --git a/libbeat/processors/actions/add_fields.go b/libbeat/processors/actions/add_fields.go index cd37910cbe5e..45f4fa60e273 100644 --- a/libbeat/processors/actions/add_fields.go +++ b/libbeat/processors/actions/add_fields.go @@ -31,7 +31,6 @@ import ( type addFields struct { fields mapstr.M - shared bool overwrite bool } @@ -61,34 +60,26 @@ func CreateAddFields(c *conf.C) (beat.Processor, error) { return makeFieldsProcessor( optTarget(config.Target, FieldsKey), config.Fields, - true, ), nil } // NewAddFields creates a new processor adding the given fields to events. -// Set `shared` true if there is the chance of labels being changed/modified by -// subsequent processors. -func NewAddFields(fields mapstr.M, shared bool, overwrite bool) beat.Processor { - return &addFields{fields: fields, shared: shared, overwrite: overwrite} +func NewAddFields(fields mapstr.M, overwrite bool) beat.Processor { + return &addFields{fields: fields, overwrite: overwrite} } -func (af *addFields) Run(event *beat.Event) (*beat.Event, error) { +func (af *addFields) Run(event *beat.EventEditor) (dropped bool, err error) { if event == nil || len(af.fields) == 0 { - return event, nil - } - - fields := af.fields - if af.shared { - fields = fields.Clone() + return false, nil } if af.overwrite { - event.DeepUpdate(fields) + event.DeepUpdate(af.fields) } else { - event.DeepUpdateNoOverwrite(fields) + event.DeepUpdateNoOverwrite(af.fields) } - return event, nil + return false, nil } func (af *addFields) String() string { @@ -103,12 +94,12 @@ func optTarget(opt *string, def string) string { return *opt } -func makeFieldsProcessor(target string, fields mapstr.M, shared bool) beat.Processor { +func makeFieldsProcessor(target string, fields mapstr.M) beat.Processor { if target != "" { fields = mapstr.M{ target: fields, } } - return NewAddFields(fields, shared, true) + return NewAddFields(fields, true) } diff --git a/libbeat/processors/actions/add_labels.go b/libbeat/processors/actions/add_labels.go index c282cf470b4d..df6c005c8ed0 100644 --- a/libbeat/processors/actions/add_labels.go +++ b/libbeat/processors/actions/add_labels.go @@ -51,7 +51,7 @@ func createAddLabels(c *conf.C) (beat.Processor, error) { return nil, fmt.Errorf("failed to flatten labels: %w", err) } - return makeFieldsProcessor(LabelsKey, flatLabels, true), nil + return makeFieldsProcessor(LabelsKey, flatLabels), nil } // NewAddLabels creates a new processor adding the given object to events. Set @@ -60,7 +60,7 @@ func createAddLabels(c *conf.C) (beat.Processor, error) { // If labels contains nested objects, NewAddLabels will flatten keys into labels by // by joining names with a dot ('.') . // The labels will be inserted into the 'labels' field. -func NewAddLabels(labels mapstr.M, shared bool) (beat.Processor, error) { +func NewAddLabels(labels mapstr.M) (beat.Processor, error) { flatLabels, err := flattenLabels(labels) if err != nil { return nil, fmt.Errorf("failed to flatten labels: %w", err) @@ -68,7 +68,7 @@ func NewAddLabels(labels mapstr.M, shared bool) (beat.Processor, error) { return NewAddFields(mapstr.M{ LabelsKey: flatLabels, - }, shared, true), nil + }, true), nil } func flattenLabels(labels mapstr.M) (mapstr.M, error) { diff --git a/libbeat/processors/actions/add_network_direction.go b/libbeat/processors/actions/add_network_direction.go index 295f74c1bef7..3681adeb4b72 100644 --- a/libbeat/processors/actions/add_network_direction.go +++ b/libbeat/processors/actions/add_network_direction.go @@ -61,45 +61,45 @@ func NewAddNetworkDirection(cfg *conf.C) (beat.Processor, error) { return networkDirection, nil } -func (m *networkDirectionProcessor) Run(event *beat.Event) (*beat.Event, error) { +func (m *networkDirectionProcessor) Run(event *beat.EventEditor) (dropped bool, err error) { sourceI, err := event.GetValue(m.Source) if err != nil { //nolint:nilerr // doesn't have the required field value to analyze - return event, nil + return false, nil } source, _ := sourceI.(string) if source == "" { // wrong type or not set - return event, nil + return false, nil } destinationI, err := event.GetValue(m.Destination) if err != nil { //nolint:nilerr // doesn't have the required field value to analyze - return event, nil + return false, nil } destination, _ := destinationI.(string) if destination == "" { // wrong type or not set - return event, nil + return false, nil } sourceIP := net.ParseIP(source) destinationIP := net.ParseIP(destination) if sourceIP == nil || destinationIP == nil { // bad ip address - return event, nil + return false, nil } internalSource, err := conditions.NetworkContains(sourceIP, m.InternalNetworks...) if err != nil { - return event, err + return false, err } internalDestination, err := conditions.NetworkContains(destinationIP, m.InternalNetworks...) if err != nil { - return event, err + return false, err } _, _ = event.PutValue(m.Target, networkDirection(internalSource, internalDestination)) - return event, nil + return false, nil } func networkDirection(internalSource, internalDestination bool) string { diff --git a/libbeat/processors/actions/add_network_direction_test.go b/libbeat/processors/actions/add_network_direction_test.go index 52ee3ba70386..060305cd4565 100644 --- a/libbeat/processors/actions/add_network_direction_test.go +++ b/libbeat/processors/actions/add_network_direction_test.go @@ -63,13 +63,15 @@ func TestNetworkDirection(t *testing.T) { "internal_networks": tt.InternalNetworks, })) require.NoError(t, err) - observed, err := p.Run(&evt) + ed := beat.NewEventEditor(&evt) + dropped, err := p.Run(ed) if tt.Error { require.Error(t, err) } else { require.NoError(t, err) + require.False(t, dropped) } - enriched, err := observed.Fields.GetValue("direction") + enriched, err := ed.GetValue("direction") if tt.Direction == "" { require.Error(t, err) } else { @@ -98,10 +100,14 @@ func TestNetworkDirection(t *testing.T) { expectedMeta := mapstr.M{ "direction": "external", } + expectedFields := evt.Fields.Clone() - observed, err := p.Run(&evt) + ed := beat.NewEventEditor(&evt) + dropped, err := p.Run(ed) require.NoError(t, err) - require.Equal(t, expectedMeta, observed.Meta) - require.Equal(t, evt.Fields, observed.Fields) + require.False(t, dropped) + ed.Apply() + require.Equal(t, expectedMeta, evt.Meta) + require.Equal(t, expectedFields, evt.Fields) }) } diff --git a/libbeat/processors/actions/add_tags.go b/libbeat/processors/actions/add_tags.go index 1cd77b7dd33f..f80ba82db765 100644 --- a/libbeat/processors/actions/add_tags.go +++ b/libbeat/processors/actions/add_tags.go @@ -73,9 +73,9 @@ func NewAddTags(target string, tags []string) beat.Processor { return &addTags{tags: tags, target: target} } -func (at *addTags) Run(event *beat.Event) (*beat.Event, error) { - _ = mapstr.AddTagsWithKey(event.Fields, at.target, at.tags) - return event, nil +func (at *addTags) Run(event *beat.EventEditor) (dropped bool, err error) { + err = event.AddTagsWithKey(at.target, at.tags...) + return false, err } func (at *addTags) String() string { diff --git a/libbeat/processors/actions/append.go b/libbeat/processors/actions/append.go index e7429449db40..d36a9b6eb8b2 100644 --- a/libbeat/processors/actions/append.go +++ b/libbeat/processors/actions/append.go @@ -73,31 +73,22 @@ func NewAppendProcessor(c *conf.C) (beat.Processor, error) { return f, nil } -func (f *appendProcessor) Run(event *beat.Event) (*beat.Event, error) { - var backup *beat.Event - if f.config.FailOnError { - backup = event.Clone() - } - - err := f.appendValues(f.config.TargetField, f.config.Fields, f.config.Values, event) +func (f *appendProcessor) Run(event *beat.EventEditor) (dropped bool, err error) { + err = f.appendValues(f.config.TargetField, f.config.Fields, f.config.Values, event) if err != nil { errMsg := fmt.Errorf("failed to append fields in append processor: %w", err) if publisher.LogWithTrace() { f.logger.Debug(errMsg.Error()) } if f.config.FailOnError { - event = backup - if _, err := event.PutValue("error.message", errMsg.Error()); err != nil { - return nil, fmt.Errorf("failed to append fields in append processor: %w", err) - } - return event, err + return false, beat.EventError{Message: errMsg.Error(), Processor: f.String()} } } - return event, nil + return false, nil } -func (f *appendProcessor) appendValues(target string, fields []string, values []interface{}, event *beat.Event) error { +func (f *appendProcessor) appendValues(target string, fields []string, values []interface{}, event *beat.EventEditor) error { var arr []interface{} // get the existing value of target field diff --git a/libbeat/processors/actions/append_test.go b/libbeat/processors/actions/append_test.go index 8cb8549b389b..2e4afca0fd5a 100644 --- a/libbeat/processors/actions/append_test.go +++ b/libbeat/processors/actions/append_test.go @@ -24,6 +24,7 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" + "github.com/stretchr/testify/assert" ) var log = logp.NewLogger("append_test") @@ -174,7 +175,7 @@ func Test_appendProcessor_appendValues(t *testing.T) { config: tt.fields.config, logger: tt.fields.logger, } - if err := f.appendValues(tt.args.target, tt.args.fields, tt.args.values, tt.args.event); (err != nil) != tt.wantErr { + if err := f.appendValues(tt.args.target, tt.args.fields, tt.args.values, beat.NewEventEditor(tt.args.event)); (err != nil) != tt.wantErr { t.Errorf("appendProcessor.appendValues() error = %v, wantErr %v", err, tt.wantErr) } }) @@ -195,6 +196,7 @@ func Test_appendProcessor_Run(t *testing.T) { args args want *beat.Event wantErr bool + wantErrMsg string }{ { description: "positive flow", @@ -378,14 +380,11 @@ func Test_appendProcessor_Run(t *testing.T) { Fields: mapstr.M{}, }, }, - wantErr: true, + wantErr: true, + wantErrMsg: "[processor=append=target] failed to append fields in append processor: could not fetch value for key: missing-field, Error: key not found", want: &beat.Event{ - Meta: mapstr.M{}, - Fields: mapstr.M{ - "error": mapstr.M{ - "message": "failed to append fields in append processor: could not fetch value for key: missing-field, Error: key not found", - }, - }, + Meta: mapstr.M{}, + Fields: mapstr.M{}, }, }, } @@ -395,14 +394,17 @@ func Test_appendProcessor_Run(t *testing.T) { config: tt.fields.config, logger: tt.fields.logger, } - got, err := f.Run(tt.args.event) - if (err != nil) != tt.wantErr { - t.Errorf("appendProcessor.Run() error = %v, wantErr %v", err, tt.wantErr) + ed := beat.NewEventEditor(tt.args.event) + _, err := f.Run(ed) + if tt.wantErr { + assert.Error(t, err) + if tt.wantErrMsg != "" { + assert.Equal(t, tt.wantErrMsg, err.Error()) + } return } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("appendProcessor.Run() = %v, want %v", got, tt.want) - } + ed.Apply() + assert.Equal(t, tt.want, tt.args.event) }) } } diff --git a/libbeat/processors/actions/common_test.go b/libbeat/processors/actions/common_test.go index e9d25708e701..f1d98e484eba 100644 --- a/libbeat/processors/actions/common_test.go +++ b/libbeat/processors/actions/common_test.go @@ -60,17 +60,18 @@ func testProcessors(t *testing.T, cases map[string]testCase) { if test.eventMeta != nil { current.Meta = test.eventMeta.Clone() } + ed := beat.NewEventEditor(current) for i, processor := range ps { var err error - current, err = processor.Run(current) + dropped, err := processor.Run(ed) if err != nil { t.Fatal(err) } - if current == nil { + if dropped { t.Fatalf("Event dropped(%v)", i) } } - + ed.Apply() assert.Equal(t, test.wantFields, current.Fields) assert.Equal(t, test.wantMeta, current.Meta) }) diff --git a/libbeat/processors/actions/copy_fields.go b/libbeat/processors/actions/copy_fields.go index 373c939388cb..a57c86ca2b40 100644 --- a/libbeat/processors/actions/copy_fields.go +++ b/libbeat/processors/actions/copy_fields.go @@ -69,12 +69,7 @@ func NewCopyFields(c *conf.C) (beat.Processor, error) { return f, nil } -func (f *copyFields) Run(event *beat.Event) (*beat.Event, error) { - var backup *beat.Event - if f.config.FailOnError { - backup = event.Clone() - } - +func (f *copyFields) Run(event *beat.EventEditor) (dropped bool, err error) { for _, field := range f.config.Fields { err := f.copyField(field.From, field.To, event) if err != nil { @@ -83,17 +78,15 @@ func (f *copyFields) Run(event *beat.Event) (*beat.Event, error) { f.logger.Debug(errMsg.Error()) } if f.config.FailOnError { - event = backup - _, _ = event.PutValue("error.message", errMsg.Error()) - return event, err + return false, beat.EventError{Message: errMsg.Error(), Processor: f.String()} } } } - return event, nil + return false, nil } -func (f *copyFields) copyField(from string, to string, event *beat.Event) error { +func (f *copyFields) copyField(from string, to string, event *beat.EventEditor) error { _, err := event.GetValue(to) if err == nil { return fmt.Errorf("target field %s already exists, drop or rename this field first", to) diff --git a/libbeat/processors/actions/copy_fields_test.go b/libbeat/processors/actions/copy_fields_test.go index 978cd24ee6a1..aa38a86f08f1 100644 --- a/libbeat/processors/actions/copy_fields_test.go +++ b/libbeat/processors/actions/copy_fields_test.go @@ -161,11 +161,12 @@ func TestCopyFields(t *testing.T) { event := &beat.Event{ Fields: test.Input, } - - newEvent, err := p.Run(event) + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) assert.NoError(t, err) - - assert.Equal(t, test.Expected, newEvent.Fields) + assert.False(t, dropped) + ed.Apply() + assert.Equal(t, test.Expected, event.Fields) }) } @@ -187,16 +188,19 @@ func TestCopyFields(t *testing.T) { "message": "please copy this line", }, } + ed := beat.NewEventEditor(event) expMeta := mapstr.M{ "message": "please copy this line", "message_copied": "please copy this line", } - newEvent, err := p.Run(event) + dropped, err := p.Run(ed) assert.NoError(t, err) + assert.False(t, dropped) - assert.Equal(t, expMeta, newEvent.Meta) - assert.Equal(t, event.Fields, newEvent.Fields) + ed.Apply() + assert.Equal(t, expMeta, event.Meta) + assert.Equal(t, event.Fields, event.Fields) }) } diff --git a/libbeat/processors/actions/decode_base64_field.go b/libbeat/processors/actions/decode_base64_field.go index fdfd038b05d6..b17c4d94b7d2 100644 --- a/libbeat/processors/actions/decode_base64_field.go +++ b/libbeat/processors/actions/decode_base64_field.go @@ -74,33 +74,26 @@ func NewDecodeBase64Field(c *cfg.C) (beat.Processor, error) { }, nil } -func (f *decodeBase64Field) Run(event *beat.Event) (*beat.Event, error) { - var backup *beat.Event - // Creates a copy of the event to revert in case of failure - if f.config.FailOnError { - backup = event.Clone() - } - - err := f.decodeField(event) +func (f *decodeBase64Field) Run(event *beat.EventEditor) (dropped bool, err error) { + err = f.decodeField(event) if err != nil { errMsg := fmt.Errorf("failed to decode base64 fields in processor: %w", err) if publisher.LogWithTrace() { f.log.Debug(errMsg.Error()) } if f.config.FailOnError { - event = backup - _, _ = event.PutValue("error.message", errMsg.Error()) - return event, err + event.AddError(beat.EventError{Message: errMsg.Error()}) + return false, err } } - return event, nil + return false, nil } func (f decodeBase64Field) String() string { return fmt.Sprintf("%s=%+v", processorName, f.config.Field) } -func (f *decodeBase64Field) decodeField(event *beat.Event) error { +func (f *decodeBase64Field) decodeField(event *beat.EventEditor) error { value, err := event.GetValue(f.config.Field.From) if err != nil { if f.config.IgnoreMissing && errors.Is(err, mapstr.ErrKeyNotFound) { diff --git a/libbeat/processors/actions/decode_base64_field_test.go b/libbeat/processors/actions/decode_base64_field_test.go index 6e6261446e33..c3c54cb3c494 100644 --- a/libbeat/processors/actions/decode_base64_field_test.go +++ b/libbeat/processors/actions/decode_base64_field_test.go @@ -211,14 +211,16 @@ func TestDecodeBase64Run(t *testing.T) { Fields: test.Input, } - newEvent, err := f.Run(event) + ed := beat.NewEventEditor(event) + dropped, err := f.Run(ed) if !test.error { assert.NoError(t, err) } else { assert.Error(t, err) } - - assert.Equal(t, test.Output, newEvent.Fields) + assert.False(t, dropped) + ed.Apply() + assert.Equal(t, test.Output, event.Fields) }) } @@ -236,6 +238,7 @@ func TestDecodeBase64Run(t *testing.T) { "field1": "Y29ycmVjdCBkYXRh", }, } + ed := beat.NewEventEditor(event) f := &decodeBase64Field{ log: logp.NewLogger(processorName), @@ -249,9 +252,12 @@ func TestDecodeBase64Run(t *testing.T) { "field": "correct data", } - newEvent, err := f.Run(event) + dropped, err := f.Run(ed) + assert.NoError(t, err) - assert.Equal(t, expectedFields, newEvent.Fields) - assert.Equal(t, expectedMeta, newEvent.Meta) + assert.False(t, dropped) + ed.Apply() + assert.Equal(t, expectedFields, event.Fields) + assert.Equal(t, expectedMeta, event.Meta) }) } diff --git a/libbeat/processors/actions/decode_json_fields.go b/libbeat/processors/actions/decode_json_fields.go index b47ebd646d9c..ca39dc3f4882 100644 --- a/libbeat/processors/actions/decode_json_fields.go +++ b/libbeat/processors/actions/decode_json_fields.go @@ -100,7 +100,7 @@ func NewDecodeJSONFields(c *cfg.C) (beat.Processor, error) { return f, nil } -func (f *decodeJSONFields) Run(event *beat.Event) (*beat.Event, error) { +func (f *decodeJSONFields) Run(event *beat.EventEditor) (dropped bool, err error) { var errs []string for _, field := range f.fields { @@ -122,7 +122,13 @@ func (f *decodeJSONFields) Run(event *beat.Event) (*beat.Event, error) { if err != nil { f.logger.Debugf("Error trying to unmarshal %s", text) errs = append(errs, err.Error()) - event.SetErrorWithOption(fmt.Sprintf("parsing input as JSON: %s", err.Error()), f.addErrorKey, text, field) + if f.addErrorKey { + event.AddError(beat.EventError{ + Message: fmt.Sprintf("parsing input as JSON: %s", err.Error()), + Data: text, + Field: field, + }) + } continue } @@ -169,17 +175,14 @@ func (f *decodeJSONFields) Run(event *beat.Event) (*beat.Event, error) { } if id != "" { - if event.Meta == nil { - event.Meta = mapstr.M{} - } - event.Meta[events.FieldMetaID] = id + _, _ = event.PutValue(beat.MetadataFieldKey+"."+events.FieldMetaID, id) } } if len(errs) > 0 { - return event, fmt.Errorf(strings.Join(errs, ", ")) + return false, fmt.Errorf(strings.Join(errs, ", ")) } - return event, nil + return false, nil } func unmarshal(maxDepth int, text string, fields *interface{}, processArray bool) error { diff --git a/libbeat/processors/actions/decode_json_fields_test.go b/libbeat/processors/actions/decode_json_fields_test.go index ba0e7fda347f..ae4ec43ec3e4 100644 --- a/libbeat/processors/actions/decode_json_fields_test.go +++ b/libbeat/processors/actions/decode_json_fields_test.go @@ -65,7 +65,7 @@ func TestMissingKey(t *testing.T) { "pipeline": "us1", } - actual := getActualValue(t, testConfig, input) + actual := getActualValue(t, testConfig, input, "") expected := mapstr.M{ "pipeline": "us1", @@ -80,7 +80,7 @@ func TestFieldNotString(t *testing.T) { "pipeline": "us1", } - actual := getActualValue(t, testConfig, input) + actual := getActualValue(t, testConfig, input, "") expected := mapstr.M{ "msg": 123, @@ -96,7 +96,7 @@ func TestInvalidJSON(t *testing.T) { "pipeline": "us1", } - actual := getActualValue(t, testConfig, input) + actual := getActualValue(t, testConfig, input, "unexpected EOF") expected := mapstr.M{ "msg": "{\"log\":\"{\\\"level\\\":\\\"info\\\"}\",\"stream\":\"stderr\",\"count\":3", @@ -111,7 +111,7 @@ func TestInvalidJSONMultiple(t *testing.T) { "pipeline": "us1", } - actual := getActualValue(t, testConfig, input) + actual := getActualValue(t, testConfig, input, "multiple json elements found") expected := mapstr.M{ "msg": "11:38:04,323 |-INFO testing", @@ -138,8 +138,11 @@ func TestDocumentID(t *testing.T) { t.Fatal(err) } - actual, err := p.Run(&beat.Event{Fields: input}) + event := &beat.Event{Fields: input} + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) require.NoError(t, err) + require.False(t, dropped) wantFields := mapstr.M{ "msg": map[string]interface{}{"log": "message"}, @@ -148,8 +151,9 @@ func TestDocumentID(t *testing.T) { "_id": "myDocumentID", } - assert.Equal(t, wantFields, actual.Fields) - assert.Equal(t, wantMeta, actual.Meta) + ed.Apply() + assert.Equal(t, wantFields, event.Fields) + assert.Equal(t, wantMeta, event.Meta) } func TestValidJSONDepthOne(t *testing.T) { @@ -158,7 +162,7 @@ func TestValidJSONDepthOne(t *testing.T) { "pipeline": "us1", } - actual := getActualValue(t, testConfig, input) + actual := getActualValue(t, testConfig, input, "") expected := mapstr.M{ "msg": map[string]interface{}{ @@ -184,7 +188,7 @@ func TestValidJSONDepthTwo(t *testing.T) { "max_depth": 2, }) - actual := getActualValue(t, testConfig, input) + actual := getActualValue(t, testConfig, input, "") expected := mapstr.M{ "msg": map[string]interface{}{ @@ -213,7 +217,7 @@ func TestTargetOption(t *testing.T) { "target": "doc", }) - actual := getActualValue(t, testConfig, input) + actual := getActualValue(t, testConfig, input, "") expected := mapstr.M{ "doc": map[string]interface{}{ @@ -243,7 +247,7 @@ func TestTargetRootOption(t *testing.T) { "target": "", }) - actual := getActualValue(t, testConfig, input) + actual := getActualValue(t, testConfig, input, "") expected := mapstr.M{ "log": map[string]interface{}{ @@ -282,7 +286,10 @@ func TestTargetMetadata(t *testing.T) { t.Fatal(err) } - actual, _ := p.Run(event) + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) + require.NoError(t, err) + require.False(t, dropped) expectedMeta := mapstr.M{ "json": map[string]interface{}{ @@ -293,9 +300,12 @@ func TestTargetMetadata(t *testing.T) { "count": int64(3), }, } + expectedFields := event.Fields.Clone() + + ed.Apply() - assert.Equal(t, expectedMeta, actual.Meta) - assert.Equal(t, event.Fields, actual.Fields) + assert.Equal(t, expectedMeta, event.Meta) + assert.Equal(t, expectedFields, event.Fields) } func TestNotJsonObjectOrArray(t *testing.T) { @@ -353,7 +363,7 @@ func TestNotJsonObjectOrArray(t *testing.T) { "max_depth": testCase.MaxDepth, }) - actual := getActualValue(t, testConfig, input) + actual := getActualValue(t, testConfig, input, "") assert.Equal(t, testCase.Expected.String(), actual.String()) }) } @@ -372,7 +382,7 @@ func TestArrayWithArraysDisabled(t *testing.T) { "process_array": false, }) - actual := getActualValue(t, testConfig, input) + actual := getActualValue(t, testConfig, input, "") expected := mapstr.M{ "msg": mapstr.M{ @@ -396,7 +406,7 @@ func TestArrayWithArraysEnabled(t *testing.T) { "process_array": true, }) - actual := getActualValue(t, testConfig, input) + actual := getActualValue(t, testConfig, input, "") expected := mapstr.M{ "msg": mapstr.M{ @@ -420,7 +430,7 @@ func TestArrayWithInvalidArray(t *testing.T) { "process_array": true, }) - actual := getActualValue(t, testConfig, input) + actual := getActualValue(t, testConfig, input, "") expected := mapstr.M{ "msg": mapstr.M{ @@ -438,7 +448,7 @@ func TestAddErrKeyOption(t *testing.T) { expectedOutput mapstr.M }{ {name: "With add_error_key option", addErrOption: true, expectedOutput: mapstr.M{ - "error": mapstr.M{"message": "@timestamp not overwritten (parse error on {})", "type": "json"}, + "error": mapstr.M{"message": "timestamp parse error on {}", "field": "@timestamp"}, "msg": "{\"@timestamp\":\"{}\"}", }}, {name: "Without add_error_key option", addErrOption: false, expectedOutput: mapstr.M{ @@ -457,7 +467,7 @@ func TestAddErrKeyOption(t *testing.T) { "overwrite_keys": true, "target": "", }) - actual := getActualValue(t, testConfig, input) + actual := getActualValue(t, testConfig, input, "") assert.Equal(t, test.expectedOutput.String(), actual.String()) @@ -481,7 +491,7 @@ func TestExpandKeys(t *testing.T) { }, }, } - actual := getActualValue(t, testConfig, input) + actual := getActualValue(t, testConfig, input, "") assert.Equal(t, expected, actual) } @@ -503,7 +513,7 @@ func TestExpandKeysWithTarget(t *testing.T) { }, }, } - actual := getActualValue(t, testConfig, input) + actual := getActualValue(t, testConfig, input, "") assert.Equal(t, expected, actual) } @@ -521,11 +531,10 @@ func TestExpandKeysError(t *testing.T) { "msg": `{"a.b": "c", "a.b.c": "d"}`, "error": mapstr.M{ "message": "cannot expand ...", - "type": "json", }, } - actual := getActualValue(t, testConfig, input) + actual := getActualValue(t, testConfig, input, "") assert.Contains(t, actual, "error") errorField := actual["error"].(mapstr.M) assert.Contains(t, errorField, "message") @@ -553,7 +562,7 @@ func TestOverwriteMetadata(t *testing.T) { expected := mapstr.M{ "msg": "overwrite metadata test", } - actual := getActualValue(t, testConfig, input) + actual := getActualValue(t, testConfig, input, "") assert.Equal(t, expected, actual) } @@ -568,7 +577,7 @@ func TestAddErrorToEventOnUnmarshalError(t *testing.T) { "message": "Broken JSON [[", } - actual := getActualValue(t, testConfig, input) + actual := getActualValue(t, testConfig, input, "invalid character 'B' looking for beginning of value") errObj, ok := actual["error"].(mapstr.M) require.True(t, ok, "'error' field not present or of invalid type") @@ -579,7 +588,7 @@ func TestAddErrorToEventOnUnmarshalError(t *testing.T) { assert.NotNil(t, errObj["message"]) } -func getActualValue(t *testing.T, config *conf.C, input mapstr.M) mapstr.M { +func getActualValue(t *testing.T, config *conf.C, input mapstr.M, expErr string) mapstr.M { log := logp.NewLogger("decode_json_fields_test") p, err := NewDecodeJSONFields(config) @@ -588,6 +597,16 @@ func getActualValue(t *testing.T, config *conf.C, input mapstr.M) mapstr.M { t.Fatal(err) } - actual, _ := p.Run(&beat.Event{Fields: input}) - return actual.Fields + event := &beat.Event{Fields: input} + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) + if expErr != "" { + require.Error(t, err) + require.Equal(t, expErr, err.Error()) + } else { + require.NoError(t, err) + } + require.False(t, dropped) + ed.Apply() + return event.Fields } diff --git a/libbeat/processors/actions/decompress_gzip_field.go b/libbeat/processors/actions/decompress_gzip_field.go index cdd731854206..7a8dc6a9cda3 100644 --- a/libbeat/processors/actions/decompress_gzip_field.go +++ b/libbeat/processors/actions/decompress_gzip_field.go @@ -67,28 +67,22 @@ func NewDecompressGzipFields(c *conf.C) (beat.Processor, error) { } // Run applies the decompress_gzip_fields processor to an event. -func (f *decompressGzipField) Run(event *beat.Event) (*beat.Event, error) { - var backup *beat.Event - if f.config.FailOnError { - backup = event.Clone() - } - - err := f.decompressGzipField(event) +func (f *decompressGzipField) Run(event *beat.EventEditor) (dropped bool, err error) { + err = f.decompressGzipField(event) if err != nil { errMsg := fmt.Errorf("Failed to decompress field in decompress_gzip_field processor: %w", err) if publisher.LogWithTrace() { f.log.Debug(errMsg.Error()) } if f.config.FailOnError { - event = backup - _, _ = event.PutValue("error.message", errMsg.Error()) - return event, err + event.AddError(beat.EventError{Message: errMsg.Error()}) + return false, err } } - return event, nil + return false, nil } -func (f *decompressGzipField) decompressGzipField(event *beat.Event) error { +func (f *decompressGzipField) decompressGzipField(event *beat.EventEditor) error { data, err := event.GetValue(f.config.Field.From) if err != nil { if f.config.IgnoreMissing && errors.Is(err, mapstr.ErrKeyNotFound) { diff --git a/libbeat/processors/actions/decompress_gzip_field_test.go b/libbeat/processors/actions/decompress_gzip_field_test.go index 230e36ecb6f5..49f89bf830be 100644 --- a/libbeat/processors/actions/decompress_gzip_field_test.go +++ b/libbeat/processors/actions/decompress_gzip_field_test.go @@ -178,14 +178,16 @@ func TestDecompressGzip(t *testing.T) { Fields: test.input, } - newEvent, err := f.Run(event) + ed := beat.NewEventEditor(event) + dropped, err := f.Run(ed) if !test.error { assert.NoError(t, err) } else { assert.Error(t, err) } - - assert.Equal(t, test.output, newEvent.Fields) + assert.False(t, dropped) + ed.Apply() + assert.Equal(t, test.output, event.Fields) }) } @@ -202,6 +204,7 @@ func TestDecompressGzip(t *testing.T) { expectedMeta := mapstr.M{ "field": "decompressed data", } + expectedFields := event.Fields.Clone() config := decompressGzipFieldConfig{ Field: fromTo{ @@ -216,10 +219,13 @@ func TestDecompressGzip(t *testing.T) { config: config, } - newEvent, err := f.Run(event) + ed := beat.NewEventEditor(event) + dropped, err := f.Run(ed) assert.NoError(t, err) + assert.False(t, dropped) - assert.Equal(t, expectedMeta, newEvent.Meta) - assert.Equal(t, event.Fields, newEvent.Fields) + ed.Apply() + assert.Equal(t, expectedMeta, event.Meta) + assert.Equal(t, expectedFields, event.Fields) }) } diff --git a/libbeat/processors/actions/detect_mime_type.go b/libbeat/processors/actions/detect_mime_type.go index 5a5ecf34d2f1..21980808f90e 100644 --- a/libbeat/processors/actions/detect_mime_type.go +++ b/libbeat/processors/actions/detect_mime_type.go @@ -49,21 +49,21 @@ func NewDetectMimeType(cfg *conf.C) (beat.Processor, error) { return mimeType, nil } -func (m *mimeTypeProcessor) Run(event *beat.Event) (*beat.Event, error) { +func (m *mimeTypeProcessor) Run(event *beat.EventEditor) (dropped bool, err error) { valI, err := event.GetValue(m.Field) if err != nil { //nolint:nilerr // doesn't have the required field value to analyze - return event, nil + return false, nil } val, _ := valI.(string) if val == "" { // wrong type or not set - return event, nil + return false, nil } if mimeType := mime.Detect(val); mimeType != "" { _, err = event.PutValue(m.Target, mimeType) } - return event, err + return false, err } func (m *mimeTypeProcessor) String() string { diff --git a/libbeat/processors/actions/detect_mime_type_test.go b/libbeat/processors/actions/detect_mime_type_test.go index 3a8a60aa282d..1272efa33fbe 100644 --- a/libbeat/processors/actions/detect_mime_type_test.go +++ b/libbeat/processors/actions/detect_mime_type_test.go @@ -28,7 +28,7 @@ import ( ) func TestMimeTypeFromTo(t *testing.T) { - evt := beat.Event{ + evt := &beat.Event{ Fields: mapstr.M{ "foo.bar.baz": "hello world!", }, @@ -38,15 +38,20 @@ func TestMimeTypeFromTo(t *testing.T) { "target": "bar.baz.zoiks", })) require.NoError(t, err) - observed, err := p.Run(&evt) + + ed := beat.NewEventEditor(evt) + dropped, err := p.Run(ed) require.NoError(t, err) - enriched, err := observed.Fields.GetValue("bar.baz.zoiks") + require.False(t, dropped) + + ed.Apply() + enriched, err := evt.Fields.GetValue("bar.baz.zoiks") require.NoError(t, err) require.Equal(t, "text/plain; charset=utf-8", enriched) } func TestMimeTypeFromToMetadata(t *testing.T) { - evt := beat.Event{ + evt := &beat.Event{ Meta: mapstr.M{}, Fields: mapstr.M{ "foo.bar.baz": "hello world!", @@ -55,20 +60,25 @@ func TestMimeTypeFromToMetadata(t *testing.T) { expectedMeta := mapstr.M{ "field": "text/plain; charset=utf-8", } + expectedFields := evt.Fields.Clone() + p, err := NewDetectMimeType(conf.MustNewConfigFrom(map[string]interface{}{ "field": "foo.bar.baz", "target": "@metadata.field", })) require.NoError(t, err) - observed, err := p.Run(&evt) + ed := beat.NewEventEditor(evt) + dropped, err := p.Run(ed) require.NoError(t, err) - require.Equal(t, expectedMeta, observed.Meta) - require.Equal(t, evt.Fields, observed.Fields) + require.False(t, dropped) + ed.Apply() + require.Equal(t, expectedMeta, evt.Meta) + require.Equal(t, expectedFields, evt.Fields) } func TestMimeTypeTestNoMatch(t *testing.T) { - evt := beat.Event{ + evt := &beat.Event{ Fields: mapstr.M{ "foo.bar.baz": string([]byte{0, 0}), }, @@ -78,8 +88,13 @@ func TestMimeTypeTestNoMatch(t *testing.T) { "target": "bar.baz.zoiks", })) require.NoError(t, err) - observed, err := p.Run(&evt) + + ed := beat.NewEventEditor(evt) + dropped, err := p.Run(ed) require.NoError(t, err) - hasKey, _ := observed.Fields.HasKey("bar.baz.zoiks") + require.False(t, dropped) + ed.Apply() + + hasKey, _ := evt.Fields.HasKey("bar.baz.zoiks") require.False(t, hasKey) } diff --git a/libbeat/processors/actions/drop_event.go b/libbeat/processors/actions/drop_event.go index bd0ef756b081..f9060ac3c92e 100644 --- a/libbeat/processors/actions/drop_event.go +++ b/libbeat/processors/actions/drop_event.go @@ -37,9 +37,8 @@ func newDropEvent(c *conf.C) (beat.Processor, error) { return dropEventsSingleton, nil } -func (*dropEvent) Run(_ *beat.Event) (*beat.Event, error) { - // return event=nil to delete the entire event - return nil, nil +func (*dropEvent) Run(*beat.EventEditor) (dropped bool, err error) { + return true, nil } func (*dropEvent) String() string { return "drop_event" } diff --git a/libbeat/processors/actions/drop_fields.go b/libbeat/processors/actions/drop_fields.go index a90d6438125b..336344a5099f 100644 --- a/libbeat/processors/actions/drop_fields.go +++ b/libbeat/processors/actions/drop_fields.go @@ -91,27 +91,51 @@ func newDropFields(c *conf.C) (beat.Processor, error) { return f, nil } -func (f *dropFields) Run(event *beat.Event) (*beat.Event, error) { +func (f *dropFields) Run(event *beat.EventEditor) (dropped bool, err error) { var errs []error + droppedKeys := make(map[string]struct{}) // remove exact match fields for _, field := range f.Fields { + if f.checkAlreadyDropped(droppedKeys, field) { + continue + } + droppedKeys[field] = struct{}{} f.deleteField(event, field, &errs) } // remove fields contained in regexp expressions - for _, regex := range f.RegexpFields { - for _, field := range *event.Fields.FlattenKeys() { - if regex.MatchString(field) { - f.deleteField(event, field, &errs) + for _, field := range event.FlattenKeys() { + if f.checkAlreadyDropped(droppedKeys, field) { + continue + } + for _, regex := range f.RegexpFields { + if !regex.MatchString(field) { + continue } + droppedKeys[field] = struct{}{} + f.deleteField(event, field, &errs) + } + } + + return false, multierr.Combine(errs...) +} + +func (f *dropFields) checkAlreadyDropped(droppedKeys map[string]struct{}, key string) bool { + _, dropped := droppedKeys[key] + if dropped { + return true + } + for droppedKey := range droppedKeys { + if strings.HasPrefix(key, droppedKey) { + return true } } - return event, multierr.Combine(errs...) + return false } -func (f *dropFields) deleteField(event *beat.Event, field string, errs *[]error) { +func (f *dropFields) deleteField(event *beat.EventEditor, field string, errs *[]error) { if err := event.Delete(field); err != nil { if !f.IgnoreMissing || !errors.Is(err, mapstr.ErrKeyNotFound) { *errs = append(*errs, fmt.Errorf("failed to drop field [%v], error: %w", field, err)) diff --git a/libbeat/processors/actions/drop_fields_test.go b/libbeat/processors/actions/drop_fields_test.go index f8a701d5af07..8c580fbd1969 100644 --- a/libbeat/processors/actions/drop_fields_test.go +++ b/libbeat/processors/actions/drop_fields_test.go @@ -44,10 +44,18 @@ func TestDropFieldRun(t *testing.T) { Fields: []string{"field"}, } - newEvent, err := p.Run(event) + cloned := &beat.Event{ + Fields: event.Fields.Clone(), + Meta: event.Meta.Clone(), + } + ed := beat.NewEventEditor(cloned) + + dropped, err := p.Run(ed) assert.NoError(t, err) - assert.Equal(t, mapstr.M{}, newEvent.Fields) - assert.Equal(t, event.Meta, newEvent.Meta) + assert.False(t, dropped) + ed.Apply() + assert.Equal(t, mapstr.M{}, cloned.Fields) + assert.Equal(t, event.Meta, cloned.Meta) }) t.Run("supports a metadata field", func(t *testing.T) { @@ -55,10 +63,18 @@ func TestDropFieldRun(t *testing.T) { Fields: []string{"@metadata.meta_field"}, } - newEvent, err := p.Run(event) + cloned := &beat.Event{ + Fields: event.Fields.Clone(), + Meta: event.Meta.Clone(), + } + ed := beat.NewEventEditor(cloned) + + dropped, err := p.Run(ed) assert.NoError(t, err) - assert.Equal(t, mapstr.M{}, newEvent.Meta) - assert.Equal(t, event.Fields, newEvent.Fields) + assert.False(t, dropped) + ed.Apply() + assert.Equal(t, mapstr.M{}, cloned.Meta) + assert.Equal(t, event.Fields, cloned.Fields) }) t.Run("supports a regexp field", func(t *testing.T) { @@ -80,17 +96,28 @@ func TestDropFieldRun(t *testing.T) { } p := dropFields{ - RegexpFields: []match.Matcher{match.MustCompile("field_2$"), match.MustCompile("field_1\\.(.*)\\.subfield_2_1"), match.MustCompile("field_1\\.subfield_3(.*)")}, - Fields: []string{}, + RegexpFields: []match.Matcher{ + match.MustCompile("field_2$"), + match.MustCompile("field_1\\.(.*)\\.subfield_2_1"), + match.MustCompile("field_1\\.subfield_3(.*)"), + }, + Fields: []string{}, } - newEvent, err := p.Run(event) - assert.NoError(t, err) - assert.Equal(t, mapstr.M{ + expFields := mapstr.M{ "field_1": mapstr.M{ "subfield_1": "sf_1_value", }, - }, newEvent.Fields) + } + + ed := beat.NewEventEditor(event) + + dropped, err := p.Run(ed) + assert.NoError(t, err) + assert.False(t, dropped) + ed.Apply() + + assert.Equal(t, expFields, event.Fields) }) } diff --git a/libbeat/processors/actions/extract_field.go b/libbeat/processors/actions/extract_field.go index 0146526939d7..04b949207bbf 100644 --- a/libbeat/processors/actions/extract_field.go +++ b/libbeat/processors/actions/extract_field.go @@ -72,26 +72,26 @@ func NewExtractField(c *conf.C) (beat.Processor, error) { return f, nil } -func (f *extract_field) Run(event *beat.Event) (*beat.Event, error) { +func (f *extract_field) Run(event *beat.EventEditor) (dropped bool, err error) { fieldValue, err := event.GetValue(f.Field) if err != nil { - return event, fmt.Errorf("error getting field '%s' from event", f.Field) + return false, fmt.Errorf("error getting field '%s' from event", f.Field) } value, ok := fieldValue.(string) if !ok { - return event, fmt.Errorf("could not get a string from field '%s'", f.Field) + return false, fmt.Errorf("could not get a string from field '%s'", f.Field) } parts := strings.Split(value, f.Separator) parts = deleteEmpty(parts) if len(parts) < f.Index+1 { - return event, fmt.Errorf("index is out of range for field '%s'", f.Field) + return false, fmt.Errorf("index is out of range for field '%s'", f.Field) } _, _ = event.PutValue(f.Target, parts[f.Index]) - return event, nil + return false, nil } func (f extract_field) String() string { diff --git a/libbeat/processors/actions/extract_field_test.go b/libbeat/processors/actions/extract_field_test.go index 2b71105c65b4..d7c58c4723a4 100644 --- a/libbeat/processors/actions/extract_field_test.go +++ b/libbeat/processors/actions/extract_field_test.go @@ -91,21 +91,18 @@ func TestCommonPaths(t *testing.T) { test.Field: test.Value, } - event, err := runExtractField(t, testConfig, input) + event, dropped, err := runExtractField(t, testConfig, input) if test.Error { assert.Error(t, err) } else { - assert.NoError(t, err) - result, err := event.Fields.GetValue(test.Target) + result, err := event.GetValue(test.Target) if err != nil { t.Fatalf("could not get target field: %s", err) } assert.Equal(t, result.(string), test.Result) } - - // Event must be present, even on error - assert.NotNil(t, event) + assert.False(t, dropped) } t.Run("supports a metadata field", func(t *testing.T) { @@ -135,14 +132,17 @@ func TestCommonPaths(t *testing.T) { t.Fatalf("error initializing extract_field: %s", err) } - newEvent, err := p.Run(event) + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) assert.NoError(t, err) - assert.Equal(t, expectedFields, newEvent.Fields) - assert.Equal(t, expectedMeta, newEvent.Meta) + assert.False(t, dropped) + ed.Apply() + assert.Equal(t, expectedFields, event.Fields) + assert.Equal(t, expectedMeta, event.Meta) }) } -func runExtractField(t *testing.T, config *conf.C, input mapstr.M) (*beat.Event, error) { +func runExtractField(t *testing.T, config *conf.C, input mapstr.M) (event *beat.Event, dropped bool, err error) { logp.TestingSetup() p, err := NewExtractField(config) @@ -150,5 +150,9 @@ func runExtractField(t *testing.T, config *conf.C, input mapstr.M) (*beat.Event, t.Fatalf("error initializing extract_field: %s", err) } - return p.Run(&beat.Event{Fields: input}) + event = &beat.Event{Fields: input} + ed := beat.NewEventEditor(event) + dropped, err = p.Run(ed) + ed.Apply() + return event, dropped, err } diff --git a/libbeat/processors/actions/include_fields.go b/libbeat/processors/actions/include_fields.go index 08419e7c2eca..83f30ade596d 100644 --- a/libbeat/processors/actions/include_fields.go +++ b/libbeat/processors/actions/include_fields.go @@ -66,7 +66,7 @@ func newIncludeFields(c *conf.C) (beat.Processor, error) { return f, nil } -func (f *includeFields) Run(event *beat.Event) (*beat.Event, error) { +func (f *includeFields) Run(event *beat.EventEditor) (dropped bool, err error) { filtered := mapstr.M{} var errs []string @@ -82,11 +82,15 @@ func (f *includeFields) Run(event *beat.Event) (*beat.Event, error) { } } - event.Fields = filtered + // does not apply the changes yet, only marks + event.DeleteAll() + for key := range filtered { + _, _ = event.PutValue(key, filtered[key]) + } if len(errs) > 0 { - return event, fmt.Errorf(strings.Join(errs, ", ")) + return false, fmt.Errorf(strings.Join(errs, ", ")) } - return event, nil + return false, nil } func (f *includeFields) String() string { diff --git a/libbeat/processors/actions/include_fields_test.go b/libbeat/processors/actions/include_fields_test.go index 9e5e6199feca..fbcb278aedc4 100644 --- a/libbeat/processors/actions/include_fields_test.go +++ b/libbeat/processors/actions/include_fields_test.go @@ -68,9 +68,11 @@ func TestIncludeFields(t *testing.T) { Fields: test.Input, } - newEvent, err := p.Run(event) + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) assert.NoError(t, err) - - assert.Equal(t, test.Output, newEvent.Fields) + assert.False(t, dropped) + ed.Apply() + assert.Equal(t, test.Output, event.Fields) } } diff --git a/libbeat/processors/actions/rename.go b/libbeat/processors/actions/rename.go index 15d2627540bf..82909c5dd915 100644 --- a/libbeat/processors/actions/rename.go +++ b/libbeat/processors/actions/rename.go @@ -73,13 +73,7 @@ func NewRenameFields(c *conf.C) (beat.Processor, error) { return f, nil } -func (f *renameFields) Run(event *beat.Event) (*beat.Event, error) { - var backup *beat.Event - // Creates a copy of the event to revert in case of failure - if f.config.FailOnError { - backup = event.Clone() - } - +func (f *renameFields) Run(event *beat.EventEditor) (dropped bool, err error) { for _, field := range f.config.Fields { err := f.renameField(field.From, field.To, event) if err != nil { @@ -88,17 +82,15 @@ func (f *renameFields) Run(event *beat.Event) (*beat.Event, error) { f.logger.Debug(errMsg.Error()) } if f.config.FailOnError { - event = backup - _, _ = event.PutValue("error.message", errMsg.Error()) - return event, err + return false, beat.EventError{Message: errMsg.Error(), Processor: f.String()} } } } - return event, nil + return false, nil } -func (f *renameFields) renameField(from string, to string, event *beat.Event) error { +func (f *renameFields) renameField(from string, to string, event *beat.EventEditor) error { // Fields cannot be overwritten. Either the target field has to be dropped first or renamed first _, err := event.GetValue(to) if err == nil { diff --git a/libbeat/processors/actions/rename_test.go b/libbeat/processors/actions/rename_test.go index 05f66eaf2ef1..ab746f058212 100644 --- a/libbeat/processors/actions/rename_test.go +++ b/libbeat/processors/actions/rename_test.go @@ -18,7 +18,6 @@ package actions import ( - "reflect" "testing" "github.com/elastic/elastic-agent-libs/logp" @@ -39,6 +38,7 @@ func TestRenameRun(t *testing.T) { Input mapstr.M Output mapstr.M error bool + expErrMsg string }{ { description: "simple field renaming", @@ -95,11 +95,9 @@ func TestRenameRun(t *testing.T) { Output: mapstr.M{ "a": 2, "b": "q", - "error": mapstr.M{ - "message": "Failed to rename fields in processor: target field b already exists, drop or rename this field first", - }, }, error: true, + expErrMsg: "[processor=rename=[{From:a To:b}]] Failed to rename fields in processor: target field b already exists, drop or rename this field first", FailOnError: true, IgnoreMissing: false, }, @@ -194,11 +192,9 @@ func TestRenameRun(t *testing.T) { Output: mapstr.M{ "a": 9, "c": 10, - "error": mapstr.M{ - "message": "Failed to rename fields in processor: could not put value: a.c: 10, expected map but type is int", - }, }, error: true, + expErrMsg: "[processor=rename=[{From:c To:a.c} {From:a To:a.value}]] Failed to rename fields in processor: could not put value: a.c: 10, expected map but type is int", IgnoreMissing: false, FailOnError: true, }, @@ -240,17 +236,25 @@ func TestRenameRun(t *testing.T) { logger: log, } event := &beat.Event{ - Fields: test.Input, + Meta: mapstr.M{}, + Fields: test.Input.Clone(), } - newEvent, err := f.Run(event) + ed := beat.NewEventEditor(event) + dropped, err := f.Run(ed) if !test.error { assert.NoError(t, err) + ed.Apply() } else { assert.Error(t, err) + if test.expErrMsg != "" { + assert.Equal(t, test.expErrMsg, err.Error()) + } + ed.Reset() } - - assert.True(t, reflect.DeepEqual(newEvent.Fields, test.Output)) + assert.False(t, dropped) + ed.Apply() + assert.Equal(t, test.Output, event.Fields) }) } } @@ -281,7 +285,7 @@ func TestRenameField(t *testing.T) { ignoreMissing: false, }, { - description: "Add hierarchy to event", + description: "add hierarchy to event", From: "a.b", To: "a.b.c", Input: mapstr.M{ @@ -358,17 +362,21 @@ func TestRenameField(t *testing.T) { }, } - err := f.renameField(test.From, test.To, &beat.Event{Fields: test.Input}) + event := &beat.Event{Fields: test.Input.Clone()} + ed := beat.NewEventEditor(event) + err := f.renameField(test.From, test.To, ed) if err != nil { assert.Equal(t, test.error, true) } - assert.True(t, reflect.DeepEqual(test.Input, test.Output)) + ed.Apply() + assert.Equal(t, test.Output, event.Fields) }) } t.Run("supports metadata as a target", func(t *testing.T) { event := &beat.Event{ + Fields: mapstr.M{}, Meta: mapstr.M{ "a": "c", }, @@ -378,6 +386,8 @@ func TestRenameField(t *testing.T) { "b": "c", } + expFields := event.Fields.Clone() + f := &renameFields{ config: renameFieldsConfig{ Fields: []fromTo{ @@ -389,9 +399,12 @@ func TestRenameField(t *testing.T) { }, } - newEvent, err := f.Run(event) + ed := beat.NewEventEditor(event) + dropped, err := f.Run(ed) assert.NoError(t, err) - assert.Equal(t, expMeta, newEvent.Meta) - assert.Equal(t, event.Fields, newEvent.Fields) + assert.False(t, dropped) + ed.Apply() + assert.Equal(t, expMeta, event.Meta) + assert.Equal(t, expFields, event.Fields) }) } diff --git a/libbeat/processors/actions/replace.go b/libbeat/processors/actions/replace.go index 07e5ba0fcc00..ef98477151fe 100644 --- a/libbeat/processors/actions/replace.go +++ b/libbeat/processors/actions/replace.go @@ -75,13 +75,7 @@ func NewReplaceString(c *conf.C) (beat.Processor, error) { return f, nil } -func (f *replaceString) Run(event *beat.Event) (*beat.Event, error) { - var backup *beat.Event - // Creates a copy of the event to revert in case of failure - if f.config.FailOnError { - backup = event.Clone() - } - +func (f *replaceString) Run(event *beat.EventEditor) (dropped bool, err error) { for _, field := range f.config.Fields { err := f.replaceField(field.Field, field.Pattern, field.Replacement, event) if err != nil { @@ -90,17 +84,15 @@ func (f *replaceString) Run(event *beat.Event) (*beat.Event, error) { f.log.Debug(errMsg.Error()) } if f.config.FailOnError { - event = backup - _, _ = event.PutValue("error.message", errMsg.Error()) - return event, err + return false, beat.EventError{Message: errMsg.Error(), Processor: f.String()} } } } - return event, nil + return false, nil } -func (f *replaceString) replaceField(field string, pattern *regexp.Regexp, replacement string, event *beat.Event) error { +func (f *replaceString) replaceField(field string, pattern *regexp.Regexp, replacement string, event *beat.EventEditor) error { currentValue, err := event.GetValue(field) if err != nil { // Ignore ErrKeyNotFound errors diff --git a/libbeat/processors/actions/replace_test.go b/libbeat/processors/actions/replace_test.go index 53977d39b92b..2f9c80563be4 100644 --- a/libbeat/processors/actions/replace_test.go +++ b/libbeat/processors/actions/replace_test.go @@ -18,7 +18,6 @@ package actions import ( - "reflect" "regexp" "testing" @@ -39,6 +38,7 @@ func TestReplaceRun(t *testing.T) { Input mapstr.M Output mapstr.M error bool + expErrMsg string }{ { description: "simple field replacing", @@ -129,11 +129,9 @@ func TestReplaceRun(t *testing.T) { Output: mapstr.M{ "m": "abc", "n": "def", - "error": mapstr.M{ - "message": "Failed to replace fields in processor: could not fetch value for key: f, Error: key not found", - }, }, error: true, + expErrMsg: "[processor=replace=[{Field:f Pattern:abc Replacement:xyz} {Field:g Pattern:def Replacement:}]] Failed to replace fields in processor: could not fetch value for key: f, Error: key not found", IgnoreMissing: false, FailOnError: true, }, @@ -153,14 +151,21 @@ func TestReplaceRun(t *testing.T) { Fields: test.Input, } - newEvent, err := f.Run(event) + ed := beat.NewEventEditor(event) + dropped, err := f.Run(ed) if !test.error { assert.NoError(t, err) + ed.Apply() } else { assert.Error(t, err) + if test.expErrMsg != "" { + assert.Equal(t, test.expErrMsg, err.Error()) + } + ed.Reset() } - - assert.True(t, reflect.DeepEqual(newEvent.Fields, test.Output)) + assert.False(t, dropped) + ed.Apply() + assert.Equal(t, test.Output, event.Fields) }) } } @@ -240,17 +245,20 @@ func TestReplaceField(t *testing.T) { }, } - err := f.replaceField(test.Field, test.Pattern, test.Replacement, &beat.Event{Fields: test.Input}) + event := &beat.Event{Fields: test.Input.Clone()} + ed := beat.NewEventEditor(event) + err := f.replaceField(test.Field, test.Pattern, test.Replacement, ed) if err != nil { assert.Equal(t, test.error, true) } - - assert.True(t, reflect.DeepEqual(test.Input, test.Output)) + ed.Apply() + assert.Equal(t, test.Output, event.Fields) }) } t.Run("supports metadata as a target", func(t *testing.T) { event := &beat.Event{ + Fields: mapstr.M{}, Meta: mapstr.M{ "f": "abc", }, @@ -260,6 +268,8 @@ func TestReplaceField(t *testing.T) { "f": "bbc", } + expectedFields := event.Fields.Clone() + f := &replaceString{ config: replaceStringConfig{ Fields: []replaceConfig{ @@ -272,9 +282,12 @@ func TestReplaceField(t *testing.T) { }, } - newEvent, err := f.Run(event) + ed := beat.NewEventEditor(event) + dropped, err := f.Run(ed) assert.NoError(t, err) - assert.Equal(t, expectedMeta, newEvent.Meta) - assert.Equal(t, event.Fields, newEvent.Fields) + assert.False(t, dropped) + ed.Apply() + assert.Equal(t, expectedMeta, event.Meta) + assert.Equal(t, expectedFields, event.Fields) }) } diff --git a/libbeat/processors/actions/truncate_fields.go b/libbeat/processors/actions/truncate_fields.go index 3eab824c99b5..7688a85f0545 100644 --- a/libbeat/processors/actions/truncate_fields.go +++ b/libbeat/processors/actions/truncate_fields.go @@ -81,27 +81,22 @@ func NewTruncateFields(c *conf.C) (beat.Processor, error) { }, nil } -func (f *truncateFields) Run(event *beat.Event) (*beat.Event, error) { - var backup *beat.Event - if f.config.FailOnError { - backup = event.Clone() - } - +func (f *truncateFields) Run(event *beat.EventEditor) (dropped bool, err error) { for _, field := range f.config.Fields { err := f.truncateSingleField(field, event) if err != nil { - f.logger.Debugf("Failed to truncate fields: %s", err) + errMsg := fmt.Errorf("Failed to truncate field: %s", err) + f.logger.Debugf(errMsg.Error()) if f.config.FailOnError { - event = backup - return event, err + return false, beat.EventError{Message: errMsg.Error(), Field: field, Processor: f.String()} } } } - return event, nil + return false, nil } -func (f *truncateFields) truncateSingleField(field string, event *beat.Event) error { +func (f *truncateFields) truncateSingleField(field string, event *beat.EventEditor) error { v, err := event.GetValue(field) if err != nil { if f.config.IgnoreMissing && errors.Is(err, mapstr.ErrKeyNotFound) { @@ -121,7 +116,7 @@ func (f *truncateFields) truncateSingleField(field string, event *beat.Event) er } -func (f *truncateFields) addTruncatedString(field, value string, event *beat.Event) error { +func (f *truncateFields) addTruncatedString(field, value string, event *beat.EventEditor) error { truncated, isTruncated, err := f.truncate(f, []byte(value)) if err != nil { return err @@ -132,12 +127,12 @@ func (f *truncateFields) addTruncatedString(field, value string, event *beat.Eve } if isTruncated { - _ = mapstr.AddTagsWithKey(event.Fields, "log.flags", []string{"truncated"}) + _ = event.AddTagsWithKey("log.flags", "truncated") } return nil } -func (f *truncateFields) addTruncatedByte(field string, value []byte, event *beat.Event) error { +func (f *truncateFields) addTruncatedByte(field string, value []byte, event *beat.EventEditor) error { truncated, isTruncated, err := f.truncate(f, value) if err != nil { return err @@ -148,7 +143,7 @@ func (f *truncateFields) addTruncatedByte(field string, value []byte, event *bea } if isTruncated { - _ = mapstr.AddTagsWithKey(event.Fields, "log.flags", []string{"truncated"}) + _ = event.AddTagsWithKey("log.flags", "truncated") } return nil } diff --git a/libbeat/processors/actions/truncate_fields_test.go b/libbeat/processors/actions/truncate_fields_test.go index 72d40649c084..a8213f790757 100644 --- a/libbeat/processors/actions/truncate_fields_test.go +++ b/libbeat/processors/actions/truncate_fields_test.go @@ -36,6 +36,7 @@ func TestTruncateFields(t *testing.T) { Input mapstr.M Output mapstr.M ShouldError bool + ExpErrMsg string TruncateFunc truncater }{ "truncate bytes of too long string line": { @@ -97,6 +98,7 @@ func TestTruncateFields(t *testing.T) { "message": 42, }, ShouldError: true, + ExpErrMsg: `[processor=truncate_fields=messagemax_bytes=5] [field="message"] Failed to truncate field: value cannot be truncated: 42`, TruncateFunc: (*truncateFields).truncateBytes, }, "do not truncate characters of short byte line": { @@ -168,14 +170,21 @@ func TestTruncateFields(t *testing.T) { Fields: test.Input, } - newEvent, err := p.Run(event) + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) if test.ShouldError { assert.Error(t, err) + if test.ExpErrMsg != "" { + assert.Equal(t, test.ExpErrMsg, err.Error()) + } + ed.Reset() } else { assert.NoError(t, err) + ed.Apply() } - - assert.Equal(t, test.Output, newEvent.Fields) + assert.False(t, dropped) + ed.Apply() + assert.Equal(t, test.Output, event.Fields) }) } @@ -207,10 +216,13 @@ func TestTruncateFields(t *testing.T) { "message": "too", } - newEvent, err := p.Run(event) + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) assert.NoError(t, err) + assert.False(t, dropped) - assert.Equal(t, expFields, newEvent.Fields) - assert.Equal(t, expMeta, newEvent.Meta) + ed.Apply() + assert.Equal(t, expFields, event.Fields) + assert.Equal(t, expMeta, event.Meta) }) } diff --git a/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go b/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go index 116c561a5097..972c9bc3ff4f 100644 --- a/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go +++ b/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go @@ -111,24 +111,24 @@ func (p *addCloudMetadata) getMeta() mapstr.M { return p.metadata.Clone() } -func (p *addCloudMetadata) Run(event *beat.Event) (*beat.Event, error) { +func (p *addCloudMetadata) Run(event *beat.EventEditor) (dropped bool, err error) { meta := p.getMeta() if len(meta) == 0 { - return event, nil + return false, nil } - err := p.addMeta(event, meta) + err = p.addMeta(event, meta) if err != nil { - return nil, err + return true, err } - return event, err + return false, err } func (p *addCloudMetadata) String() string { return "add_cloud_metadata=" + p.getMeta().String() } -func (p *addCloudMetadata) addMeta(event *beat.Event, meta mapstr.M) error { +func (p *addCloudMetadata) addMeta(event *beat.EventEditor, meta mapstr.M) error { for key, metaVal := range meta { // If key exists in event already and overwrite flag is set to false, this processor will not overwrite the // meta fields. For example aws module writes cloud.instance.* to events already, with overwrite=false, diff --git a/libbeat/processors/add_cloud_metadata/provider_alibaba_cloud_test.go b/libbeat/processors/add_cloud_metadata/provider_alibaba_cloud_test.go index 7ad595ab8e91..ae3f31a048ec 100644 --- a/libbeat/processors/add_cloud_metadata/provider_alibaba_cloud_test.go +++ b/libbeat/processors/add_cloud_metadata/provider_alibaba_cloud_test.go @@ -69,10 +69,11 @@ func TestRetrieveAlibabaCloudMetadata(t *testing.T) { t.Fatal(err) } - actual, err := p.Run(&beat.Event{Fields: mapstr.M{}}) - if err != nil { - t.Fatal(err) - } + event := &beat.Event{Fields: mapstr.M{}} + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) + assert.NoError(t, err) + assert.False(t, dropped) expected := mapstr.M{ "cloud": mapstr.M{ @@ -87,5 +88,6 @@ func TestRetrieveAlibabaCloudMetadata(t *testing.T) { }, }, } - assert.Equal(t, expected, actual.Fields) + ed.Apply() + assert.Equal(t, expected, event.Fields) } diff --git a/libbeat/processors/add_cloud_metadata/provider_aws_ec2_test.go b/libbeat/processors/add_cloud_metadata/provider_aws_ec2_test.go index 92e3ae7ec2e3..3cb14f27d9cd 100644 --- a/libbeat/processors/add_cloud_metadata/provider_aws_ec2_test.go +++ b/libbeat/processors/add_cloud_metadata/provider_aws_ec2_test.go @@ -359,11 +359,14 @@ func TestRetrieveAWSMetadataEC2(t *testing.T) { t.Fatalf("error creating new metadata processor: %s", err.Error()) } - actual, err := cmp.Run(&beat.Event{Fields: tc.previousEvent}) - if err != nil { - t.Fatalf("error running processor: %s", err.Error()) - } - assert.Equal(t, tc.expectedEvent, actual.Fields) + event := &beat.Event{Fields: tc.previousEvent} + ed := beat.NewEventEditor(event) + dropped, err := cmp.Run(ed) + assert.NoError(t, err) + assert.False(t, dropped) + + ed.Apply() + assert.Equal(t, tc.expectedEvent, event.Fields) }) } } diff --git a/libbeat/processors/add_cloud_metadata/provider_azure_vm_test.go b/libbeat/processors/add_cloud_metadata/provider_azure_vm_test.go index 30b663bdfc54..439ba290fe5d 100644 --- a/libbeat/processors/add_cloud_metadata/provider_azure_vm_test.go +++ b/libbeat/processors/add_cloud_metadata/provider_azure_vm_test.go @@ -105,10 +105,11 @@ func TestRetrieveAzureMetadata(t *testing.T) { t.Fatal(err) } - actual, err := p.Run(&beat.Event{Fields: mapstr.M{}}) - if err != nil { - t.Fatal(err) - } + event := &beat.Event{Fields: mapstr.M{}} + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) + assert.NoError(t, err) + assert.False(t, dropped) expected := mapstr.M{ "cloud": mapstr.M{ @@ -129,5 +130,6 @@ func TestRetrieveAzureMetadata(t *testing.T) { "region": "eastus", }, } - assert.Equal(t, expected, actual.Fields) + ed.Apply() + assert.Equal(t, expected, event.Fields) } diff --git a/libbeat/processors/add_cloud_metadata/provider_digital_ocean_test.go b/libbeat/processors/add_cloud_metadata/provider_digital_ocean_test.go index 3160d8eb82d9..f149673fe3a9 100644 --- a/libbeat/processors/add_cloud_metadata/provider_digital_ocean_test.go +++ b/libbeat/processors/add_cloud_metadata/provider_digital_ocean_test.go @@ -107,10 +107,11 @@ func TestRetrieveDigitalOceanMetadata(t *testing.T) { t.Fatal(err) } - actual, err := p.Run(&beat.Event{Fields: mapstr.M{}}) - if err != nil { - t.Fatal(err) - } + event := &beat.Event{Fields: mapstr.M{}} + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) + assert.NoError(t, err) + assert.False(t, dropped) expected := mapstr.M{ "cloud": mapstr.M{ @@ -124,5 +125,6 @@ func TestRetrieveDigitalOceanMetadata(t *testing.T) { "region": "nyc3", }, } - assert.Equal(t, expected, actual.Fields) + ed.Apply() + assert.Equal(t, expected, event.Fields) } diff --git a/libbeat/processors/add_cloud_metadata/provider_google_gce_test.go b/libbeat/processors/add_cloud_metadata/provider_google_gce_test.go index 97078f9a2802..c24ad67e0ad2 100644 --- a/libbeat/processors/add_cloud_metadata/provider_google_gce_test.go +++ b/libbeat/processors/add_cloud_metadata/provider_google_gce_test.go @@ -325,10 +325,11 @@ func TestRetrieveGCEMetadata(t *testing.T) { t.Fatal(err) } - actual, err := p.Run(&beat.Event{Fields: mapstr.M{}}) - if err != nil { - t.Fatal(err) - } + event := &beat.Event{Fields: mapstr.M{}} + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) + assert.NoError(t, err) + assert.False(t, dropped) expected := mapstr.M{ "cloud": mapstr.M{ @@ -353,7 +354,8 @@ func TestRetrieveGCEMetadata(t *testing.T) { }, }, } - assert.Equal(t, expected, actual.Fields) + ed.Apply() + assert.Equal(t, expected, event.Fields) } func TestRetrieveGCEMetadataInK8s(t *testing.T) { @@ -374,10 +376,11 @@ func TestRetrieveGCEMetadataInK8s(t *testing.T) { t.Fatal(err) } - actual, err := p.Run(&beat.Event{Fields: mapstr.M{}}) - if err != nil { - t.Fatal(err) - } + event := &beat.Event{Fields: mapstr.M{}} + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) + assert.NoError(t, err) + assert.False(t, dropped) expected := mapstr.M{ "cloud": mapstr.M{ @@ -408,7 +411,8 @@ func TestRetrieveGCEMetadataInK8s(t *testing.T) { }, }, } - assert.Equal(t, expected, actual.Fields) + ed.Apply() + assert.Equal(t, expected, event.Fields) } func TestRetrieveGCEMetadataInK8sNotOverriden(t *testing.T) { @@ -429,21 +433,20 @@ func TestRetrieveGCEMetadataInK8sNotOverriden(t *testing.T) { t.Fatal(err) } - actual, err := p.Run( - &beat.Event{ - Fields: mapstr.M{ - "orchestrator": mapstr.M{ - "cluster": mapstr.M{ - "name": "production-marketing-k8s", - "url": "https://35.223.150.35", - }, + event := &beat.Event{ + Fields: mapstr.M{ + "orchestrator": mapstr.M{ + "cluster": mapstr.M{ + "name": "production-marketing-k8s", + "url": "https://35.223.150.35", }, }, }, - ) - if err != nil { - t.Fatal(err) } + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) + assert.NoError(t, err) + assert.False(t, dropped) expected := mapstr.M{ "cloud": mapstr.M{ @@ -474,7 +477,8 @@ func TestRetrieveGCEMetadataInK8sNotOverriden(t *testing.T) { }, }, } - assert.Equal(t, expected, actual.Fields) + ed.Apply() + assert.Equal(t, expected, event.Fields) } func TestRetrieveGCEMetadataInK8sPartial(t *testing.T) { @@ -495,10 +499,11 @@ func TestRetrieveGCEMetadataInK8sPartial(t *testing.T) { t.Fatal(err) } - actual, err := p.Run(&beat.Event{Fields: mapstr.M{}}) - if err != nil { - t.Fatal(err) - } + event := &beat.Event{Fields: mapstr.M{}} + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) + assert.NoError(t, err) + assert.False(t, dropped) expected := mapstr.M{ "cloud": mapstr.M{ @@ -528,5 +533,6 @@ func TestRetrieveGCEMetadataInK8sPartial(t *testing.T) { }, }, } - assert.Equal(t, expected, actual.Fields) + ed.Apply() + assert.Equal(t, expected, event.Fields) } diff --git a/libbeat/processors/add_cloud_metadata/provider_hetzner_cloud_test.go b/libbeat/processors/add_cloud_metadata/provider_hetzner_cloud_test.go index f62f6629c2c9..387f56ff709c 100644 --- a/libbeat/processors/add_cloud_metadata/provider_hetzner_cloud_test.go +++ b/libbeat/processors/add_cloud_metadata/provider_hetzner_cloud_test.go @@ -76,10 +76,11 @@ func assertHetzner(t *testing.T, config *conf.C) { t.Fatal(err) } - actual, err := p.Run(&beat.Event{Fields: mapstr.M{}}) - if err != nil { - t.Fatal(err) - } + event := &beat.Event{Fields: mapstr.M{}} + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) + assert.NoError(t, err) + assert.False(t, dropped) expected := mapstr.M{ "cloud": mapstr.M{ @@ -95,5 +96,6 @@ func assertHetzner(t *testing.T, config *conf.C) { }, }, } - assert.Equal(t, expected, actual.Fields) + ed.Apply() + assert.Equal(t, expected, event.Fields) } diff --git a/libbeat/processors/add_cloud_metadata/provider_huawei_cloud_test.go b/libbeat/processors/add_cloud_metadata/provider_huawei_cloud_test.go index 0ae6fc332f09..9de85ac1aa6f 100644 --- a/libbeat/processors/add_cloud_metadata/provider_huawei_cloud_test.go +++ b/libbeat/processors/add_cloud_metadata/provider_huawei_cloud_test.go @@ -76,10 +76,11 @@ func TestRetrieveHuaweiCloudMetadata(t *testing.T) { t.Fatal(err) } - actual, err := p.Run(&beat.Event{Fields: mapstr.M{}}) - if err != nil { - t.Fatal(err) - } + event := &beat.Event{Fields: mapstr.M{}} + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) + assert.NoError(t, err) + assert.False(t, dropped) expected := mapstr.M{ "cloud": mapstr.M{ @@ -94,5 +95,6 @@ func TestRetrieveHuaweiCloudMetadata(t *testing.T) { }, }, } - assert.Equal(t, expected, actual.Fields) + ed.Apply() + assert.Equal(t, expected, event.Fields) } diff --git a/libbeat/processors/add_cloud_metadata/provider_openstack_nova_test.go b/libbeat/processors/add_cloud_metadata/provider_openstack_nova_test.go index 09c52d150665..3df1d5b38e8b 100644 --- a/libbeat/processors/add_cloud_metadata/provider_openstack_nova_test.go +++ b/libbeat/processors/add_cloud_metadata/provider_openstack_nova_test.go @@ -94,10 +94,11 @@ func assertOpenstackNova(t *testing.T, config *conf.C) { t.Fatal(err) } - actual, err := p.Run(&beat.Event{Fields: mapstr.M{}}) - if err != nil { - t.Fatal(err) - } + event := &beat.Event{Fields: mapstr.M{}} + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) + assert.NoError(t, err) + assert.False(t, dropped) expected := mapstr.M{ "cloud": mapstr.M{ @@ -115,5 +116,6 @@ func assertOpenstackNova(t *testing.T, config *conf.C) { }, }, } - assert.Equal(t, expected, actual.Fields) + ed.Apply() + assert.Equal(t, expected, event.Fields) } diff --git a/libbeat/processors/add_cloud_metadata/provider_tencent_cloud_test.go b/libbeat/processors/add_cloud_metadata/provider_tencent_cloud_test.go index 304fc492e20f..d70a743ccf4c 100644 --- a/libbeat/processors/add_cloud_metadata/provider_tencent_cloud_test.go +++ b/libbeat/processors/add_cloud_metadata/provider_tencent_cloud_test.go @@ -69,10 +69,11 @@ func TestRetrieveQCloudMetadata(t *testing.T) { t.Fatal(err) } - actual, err := p.Run(&beat.Event{Fields: mapstr.M{}}) - if err != nil { - t.Fatal(err) - } + event := &beat.Event{Fields: mapstr.M{}} + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) + assert.NoError(t, err) + assert.False(t, dropped) expected := mapstr.M{ "cloud": mapstr.M{ @@ -87,5 +88,6 @@ func TestRetrieveQCloudMetadata(t *testing.T) { }, }, } - assert.Equal(t, expected, actual.Fields) + ed.Apply() + assert.Equal(t, expected, event.Fields) } diff --git a/libbeat/processors/add_docker_metadata/add_docker_metadata.go b/libbeat/processors/add_docker_metadata/add_docker_metadata.go index d670713894da..9e3c48afb598 100644 --- a/libbeat/processors/add_docker_metadata/add_docker_metadata.go +++ b/libbeat/processors/add_docker_metadata/add_docker_metadata.go @@ -133,21 +133,21 @@ func lazyCgroupCacheInit(d *addDockerMetadata) { } } -func (d *addDockerMetadata) Run(event *beat.Event) (*beat.Event, error) { +func (d *addDockerMetadata) Run(event *beat.EventEditor) (dropped bool, err error) { if !d.dockerAvailable { - return event, nil + return false, nil } var cid string - var err error // Extract CID from the filepath contained in the "log.file.path" field. if d.sourceProcessor != nil { - lfp, _ := event.Fields.GetValue("log.file.path") + lfp, _ := event.GetValue("log.file.path") if lfp != nil { - event, err = d.sourceProcessor.Run(event) + // should never drop events + _, err = d.sourceProcessor.Run(event) if err != nil { d.log.Debugf("Error while extracting container ID from source path: %v", err) - return event, nil + return false, nil } if v, err := event.GetValue(dockerContainerIDKey); err == nil { @@ -160,7 +160,7 @@ func (d *addDockerMetadata) Run(event *beat.Event) (*beat.Event, error) { if cid == "" && len(d.pidFields) > 0 { id, err := d.lookupContainerIDByPID(event) if err != nil { - return nil, fmt.Errorf("error reading container ID: %w", err) + return true, fmt.Errorf("error reading container ID: %w", err) } if id != "" { cid = id @@ -184,7 +184,7 @@ func (d *addDockerMetadata) Run(event *beat.Event) (*beat.Event, error) { } if cid == "" { - return event, nil + return false, nil } container := d.watcher.Container(cid) @@ -207,12 +207,13 @@ func (d *addDockerMetadata) Run(event *beat.Event) (*beat.Event, error) { _, _ = meta.Put("container.id", container.ID) _, _ = meta.Put("container.image.name", container.Image) _, _ = meta.Put("container.name", container.Name) - event.Fields.DeepUpdate(meta.Clone()) + + event.DeepUpdate(meta.Clone()) } else { d.log.Debugf("Container not found: cid=%s", cid) } - return event, nil + return false, nil } func (d *addDockerMetadata) Close() error { @@ -237,7 +238,7 @@ func (d *addDockerMetadata) String() string { // lookupContainerIDByPID finds the container ID based on PID fields contained // in the event. -func (d *addDockerMetadata) lookupContainerIDByPID(event *beat.Event) (string, error) { +func (d *addDockerMetadata) lookupContainerIDByPID(event *beat.EventEditor) (string, error) { var cgroups cgroup.PathList for _, field := range d.pidFields { v, err := event.GetValue(field) diff --git a/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go b/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go index 2b6663f71dc1..e273e7002449 100644 --- a/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go +++ b/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go @@ -71,10 +71,13 @@ func TestInitializationNoDocker(t *testing.T) { assert.NoError(t, err, "initializing add_docker_metadata processor") input := mapstr.M{} - result, err := p.Run(&beat.Event{Fields: input}) + event := &beat.Event{Fields: input} + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) assert.NoError(t, err, "processing an event") - - assert.Equal(t, mapstr.M{}, result.Fields) + assert.False(t, dropped) + ed.Apply() + assert.Equal(t, mapstr.M{}, event.Fields) } func TestInitialization(t *testing.T) { @@ -84,10 +87,13 @@ func TestInitialization(t *testing.T) { assert.NoError(t, err, "initializing add_docker_metadata processor") input := mapstr.M{} - result, err := p.Run(&beat.Event{Fields: input}) + event := &beat.Event{Fields: input} + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) assert.NoError(t, err, "processing an event") - - assert.Equal(t, mapstr.M{}, result.Fields) + assert.False(t, dropped) + ed.Apply() + assert.Equal(t, mapstr.M{}, event.Fields) } func TestNoMatch(t *testing.T) { @@ -102,10 +108,13 @@ func TestNoMatch(t *testing.T) { input := mapstr.M{ "field": "value", } - result, err := p.Run(&beat.Event{Fields: input}) + event := &beat.Event{Fields: input} + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) assert.NoError(t, err, "processing an event") - - assert.Equal(t, mapstr.M{"field": "value"}, result.Fields) + assert.False(t, dropped) + ed.Apply() + assert.Equal(t, mapstr.M{"field": "value"}, event.Fields) } func TestMatchNoContainer(t *testing.T) { @@ -120,10 +129,13 @@ func TestMatchNoContainer(t *testing.T) { input := mapstr.M{ "foo": "garbage", } - result, err := p.Run(&beat.Event{Fields: input}) + event := &beat.Event{Fields: input} + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) assert.NoError(t, err, "processing an event") - - assert.Equal(t, mapstr.M{"foo": "garbage"}, result.Fields) + assert.False(t, dropped) + ed.Apply() + assert.Equal(t, mapstr.M{"foo": "garbage"}, event.Fields) } func TestMatchContainer(t *testing.T) { @@ -151,9 +163,12 @@ func TestMatchContainer(t *testing.T) { input := mapstr.M{ "foo": "container_id", } - result, err := p.Run(&beat.Event{Fields: input}) + event := &beat.Event{Fields: input} + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) assert.NoError(t, err, "processing an event") - + assert.False(t, dropped) + ed.Apply() assert.EqualValues(t, mapstr.M{ "container": mapstr.M{ "id": "container_id", @@ -172,7 +187,7 @@ func TestMatchContainer(t *testing.T) { "name": "name", }, "foo": "container_id", - }, result.Fields) + }, event.Fields) } func TestMatchContainerWithDedot(t *testing.T) { @@ -199,9 +214,12 @@ func TestMatchContainerWithDedot(t *testing.T) { input := mapstr.M{ "foo": "container_id", } - result, err := p.Run(&beat.Event{Fields: input}) + event := &beat.Event{Fields: input} + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) assert.NoError(t, err, "processing an event") - + assert.False(t, dropped) + ed.Apply() assert.EqualValues(t, mapstr.M{ "container": mapstr.M{ "id": "container_id", @@ -216,7 +234,7 @@ func TestMatchContainerWithDedot(t *testing.T) { "name": "name", }, "foo": "container_id", - }, result.Fields) + }, event.Fields) } func TestMatchSource(t *testing.T) { @@ -253,9 +271,12 @@ func TestMatchSource(t *testing.T) { }, } - result, err := p.Run(&beat.Event{Fields: input}) + event := &beat.Event{Fields: input} + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) assert.NoError(t, err, "processing an event") - + assert.False(t, dropped) + ed.Apply() assert.EqualValues(t, mapstr.M{ "container": mapstr.M{ "id": "8c147fdfab5a2608fe513d10294bf77cb502a231da9725093a155bd25cd1f14b", @@ -273,7 +294,7 @@ func TestMatchSource(t *testing.T) { "path": inputSource, }, }, - }, result.Fields) + }, event.Fields) } func TestDisableSource(t *testing.T) { @@ -300,11 +321,14 @@ func TestDisableSource(t *testing.T) { input := mapstr.M{ "source": "/var/lib/docker/containers/8c147fdfab5a2608fe513d10294bf77cb502a231da9725093a155bd25cd1f14b/foo.log", } - result, err := p.Run(&beat.Event{Fields: input}) + event := &beat.Event{Fields: input} + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) assert.NoError(t, err, "processing an event") - + assert.False(t, dropped) + ed.Apply() // remains unchanged - assert.EqualValues(t, input, result.Fields) + assert.EqualValues(t, input, event.Fields) } func TestMatchPIDs(t *testing.T) { @@ -345,9 +369,13 @@ func TestMatchPIDs(t *testing.T) { expected := mapstr.M{} expected.DeepUpdate(input) - result, err := p.Run(&beat.Event{Fields: input}) + event := &beat.Event{Fields: input} + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) assert.NoError(t, err, "processing an event") - assert.EqualValues(t, expected, result.Fields) + assert.False(t, dropped) + ed.Apply() + assert.EqualValues(t, expected, event.Fields) }) t.Run("pid does not exist", func(t *testing.T) { @@ -357,9 +385,13 @@ func TestMatchPIDs(t *testing.T) { expected := mapstr.M{} expected.DeepUpdate(input) - result, err := p.Run(&beat.Event{Fields: input}) + event := &beat.Event{Fields: input} + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) assert.NoError(t, err, "processing an event") - assert.EqualValues(t, expected, result.Fields) + assert.False(t, dropped) + ed.Apply() + assert.EqualValues(t, expected, event.Fields) }) t.Run("pid is containerized", func(t *testing.T) { @@ -370,9 +402,13 @@ func TestMatchPIDs(t *testing.T) { expected.DeepUpdate(dockerMetadata) expected.DeepUpdate(fields) - result, err := p.Run(&beat.Event{Fields: fields}) + event := &beat.Event{Fields: fields} + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) assert.NoError(t, err, "processing an event") - assert.EqualValues(t, expected, result.Fields) + assert.False(t, dropped) + ed.Apply() + assert.EqualValues(t, expected, event.Fields) }) t.Run("pid exited and ppid is containerized", func(t *testing.T) { @@ -384,9 +420,13 @@ func TestMatchPIDs(t *testing.T) { expected.DeepUpdate(dockerMetadata) expected.DeepUpdate(fields) - result, err := p.Run(&beat.Event{Fields: fields}) + event := &beat.Event{Fields: fields} + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) assert.NoError(t, err, "processing an event") - assert.EqualValues(t, expected, result.Fields) + assert.False(t, dropped) + ed.Apply() + assert.EqualValues(t, expected, event.Fields) }) t.Run("cgroup error", func(t *testing.T) { @@ -396,9 +436,13 @@ func TestMatchPIDs(t *testing.T) { expected := mapstr.M{} expected.DeepUpdate(fields) - result, err := p.Run(&beat.Event{Fields: fields}) + event := &beat.Event{Fields: fields} + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) assert.NoError(t, err, "processing an event") - assert.EqualValues(t, expected, result.Fields) + assert.False(t, dropped) + ed.Apply() + assert.EqualValues(t, expected, event.Fields) }) } diff --git a/libbeat/processors/add_host_metadata/add_host_metadata.go b/libbeat/processors/add_host_metadata/add_host_metadata.go index db3cbbc5ee30..7bea25fec559 100644 --- a/libbeat/processors/add_host_metadata/add_host_metadata.go +++ b/libbeat/processors/add_host_metadata/add_host_metadata.go @@ -121,23 +121,23 @@ func New(cfg *config.C) (beat.Processor, error) { } // Run enriches the given event with the host metadata -func (p *addHostMetadata) Run(event *beat.Event) (*beat.Event, error) { +func (p *addHostMetadata) Run(event *beat.EventEditor) (dropped bool, err error) { // check replace_host_fields field if !p.config.ReplaceFields && skipAddingHostMetadata(event) { - return event, nil + return false, nil } - err := p.loadData(true, features.FQDN()) + err = p.loadData(true, features.FQDN()) if err != nil { - return nil, fmt.Errorf("error loading data during event update: %w", err) + return true, fmt.Errorf("error loading data during event update: %w", err) } - event.Fields.DeepUpdate(p.data.Get().Clone()) + event.DeepUpdate(p.data.Get().Clone()) if len(p.geoData) > 0 { - event.Fields.DeepUpdate(p.geoData) + event.DeepUpdate(p.geoData) } - return event, nil + return false, nil } // Ideally we'd be able to implement the Closer interface here and @@ -283,9 +283,9 @@ func (p *addHostMetadata) updateOrExpire(useFQDN bool) { } -func skipAddingHostMetadata(event *beat.Event) bool { +func skipAddingHostMetadata(event *beat.EventEditor) bool { // If host fields exist(besides host.name added by libbeat) in event, skip add_host_metadata. - hostFields, err := event.Fields.GetValue("host") + hostFields, err := event.GetValue("host") // Don't skip if there are no fields if err != nil || hostFields == nil { diff --git a/libbeat/processors/add_host_metadata/add_host_metadata_test.go b/libbeat/processors/add_host_metadata/add_host_metadata_test.go index c90feb771851..50868030b258 100644 --- a/libbeat/processors/add_host_metadata/add_host_metadata_test.go +++ b/libbeat/processors/add_host_metadata/add_host_metadata_test.go @@ -60,30 +60,32 @@ func TestConfigDefault(t *testing.T) { return } - newEvent, err := p.Run(event) + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) assert.NoError(t, err) - - v, err := newEvent.GetValue("host.os.family") + assert.False(t, dropped) + ed.Apply() + v, err := event.GetValue("host.os.family") assert.NoError(t, err) assert.NotNil(t, v) - v, err = newEvent.GetValue("host.os.kernel") + v, err = event.GetValue("host.os.kernel") assert.NoError(t, err) assert.NotNil(t, v) - v, err = newEvent.GetValue("host.os.name") + v, err = event.GetValue("host.os.name") assert.NoError(t, err) assert.NotNil(t, v) - v, err = newEvent.GetValue("host.ip") + v, err = event.GetValue("host.ip") assert.NoError(t, err) assert.NotNil(t, v) - v, err = newEvent.GetValue("host.mac") + v, err = event.GetValue("host.mac") assert.NoError(t, err) assert.NotNil(t, v) - v, err = newEvent.GetValue("host.os.type") + v, err = event.GetValue("host.os.type") assert.NoError(t, err) assert.NotNil(t, v) } @@ -107,30 +109,33 @@ func TestConfigNetInfoDisabled(t *testing.T) { return } - newEvent, err := p.Run(event) + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) assert.NoError(t, err) + assert.False(t, dropped) + ed.Apply() - v, err := newEvent.GetValue("host.os.family") + v, err := event.GetValue("host.os.family") assert.NoError(t, err) assert.NotNil(t, v) - v, err = newEvent.GetValue("host.os.kernel") + v, err = event.GetValue("host.os.kernel") assert.NoError(t, err) assert.NotNil(t, v) - v, err = newEvent.GetValue("host.os.name") + v, err = event.GetValue("host.os.name") assert.NoError(t, err) assert.NotNil(t, v) - v, err = newEvent.GetValue("host.ip") + v, err = event.GetValue("host.ip") assert.Error(t, err) assert.Nil(t, v) - v, err = newEvent.GetValue("host.mac") + v, err = event.GetValue("host.mac") assert.Error(t, err) assert.Nil(t, v) - v, err = newEvent.GetValue("host.os.type") + v, err = event.GetValue("host.os.type") assert.NoError(t, err) assert.NotNil(t, v) } @@ -151,14 +156,17 @@ func TestConfigName(t *testing.T) { p, err := New(testConfig) require.NoError(t, err) - newEvent, err := p.Run(event) + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) assert.NoError(t, err) + assert.False(t, dropped) + ed.Apply() for configKey, configValue := range config { t.Run(fmt.Sprintf("Check of %s", configKey), func(t *testing.T) { - v, err := newEvent.GetValue(fmt.Sprintf("host.%s", configKey)) + v, err := event.GetValue(fmt.Sprintf("host.%s", configKey)) assert.NoError(t, err) - assert.Equal(t, configValue, v, "Could not find in %s", newEvent) + assert.Equal(t, configValue, v, "Could not find in %s", event) }) } } @@ -186,10 +194,13 @@ func TestConfigGeoEnabled(t *testing.T) { p, err := New(testConfig) require.NoError(t, err) - newEvent, err := p.Run(event) + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) assert.NoError(t, err) + assert.False(t, dropped) + ed.Apply() - eventGeoField, err := newEvent.GetValue("host.geo") + eventGeoField, err := event.GetValue("host.geo") require.NoError(t, err) assert.Len(t, eventGeoField, len(config)) @@ -209,11 +220,13 @@ func TestConfigGeoDisabled(t *testing.T) { p, err := New(testConfig) require.NoError(t, err) - newEvent, err := p.Run(event) - - require.NoError(t, err) + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) + assert.NoError(t, err) + assert.False(t, dropped) + ed.Apply() - eventGeoField, err := newEvent.GetValue("host.geo") + eventGeoField, err := event.GetValue("host.geo") assert.Error(t, err) assert.Equal(t, nil, eventGeoField) } @@ -235,14 +248,14 @@ func TestEventWithReplaceFieldsFalse(t *testing.T) { cases := []struct { title string - event beat.Event + event *beat.Event hostLengthLargerThanOne bool hostLengthEqualsToOne bool expectedHostFieldLength int }{ { "replace_fields=false with only host.name", - beat.Event{ + &beat.Event{ Fields: mapstr.M{ "host": mapstr.M{ "name": hostName, @@ -255,7 +268,7 @@ func TestEventWithReplaceFieldsFalse(t *testing.T) { }, { "replace_fields=false with only host.id", - beat.Event{ + &beat.Event{ Fields: mapstr.M{ "host": mapstr.M{ "id": hostID, @@ -268,7 +281,7 @@ func TestEventWithReplaceFieldsFalse(t *testing.T) { }, { "replace_fields=false with host.name and host.id", - beat.Event{ + &beat.Event{ Fields: mapstr.M{ "host": mapstr.M{ "name": hostName, @@ -284,10 +297,13 @@ func TestEventWithReplaceFieldsFalse(t *testing.T) { for _, c := range cases { t.Run(c.title, func(t *testing.T) { - newEvent, err := p.Run(&c.event) + ed := beat.NewEventEditor(c.event) + dropped, err := p.Run(ed) assert.NoError(t, err) + assert.False(t, dropped) + ed.Apply() - v, err := newEvent.GetValue("host") + v, err := c.event.GetValue("host") assert.NoError(t, err) assert.Equal(t, c.hostLengthLargerThanOne, len(v.(mapstr.M)) > 1) assert.Equal(t, c.hostLengthEqualsToOne, len(v.(mapstr.M)) == 1) @@ -315,13 +331,13 @@ func TestEventWithReplaceFieldsTrue(t *testing.T) { cases := []struct { title string - event beat.Event + event *beat.Event hostLengthLargerThanOne bool hostLengthEqualsToOne bool }{ { "replace_fields=true with host.name", - beat.Event{ + &beat.Event{ Fields: mapstr.M{ "host": mapstr.M{ "name": hostName, @@ -333,7 +349,7 @@ func TestEventWithReplaceFieldsTrue(t *testing.T) { }, { "replace_fields=true with host.id", - beat.Event{ + &beat.Event{ Fields: mapstr.M{ "host": mapstr.M{ "id": hostID, @@ -345,7 +361,7 @@ func TestEventWithReplaceFieldsTrue(t *testing.T) { }, { "replace_fields=true with host.name and host.id", - beat.Event{ + &beat.Event{ Fields: mapstr.M{ "host": mapstr.M{ "name": hostName, @@ -360,10 +376,13 @@ func TestEventWithReplaceFieldsTrue(t *testing.T) { for _, c := range cases { t.Run(c.title, func(t *testing.T) { - newEvent, err := p.Run(&c.event) + ed := beat.NewEventEditor(c.event) + dropped, err := p.Run(ed) assert.NoError(t, err) + assert.False(t, dropped) + ed.Apply() - v, err := newEvent.GetValue("host") + v, err := c.event.GetValue("host") assert.NoError(t, err) assert.Equal(t, c.hostLengthLargerThanOne, len(v.(mapstr.M)) > 1) assert.Equal(t, c.hostLengthEqualsToOne, len(v.(mapstr.M)) == 1) @@ -384,12 +403,12 @@ func TestSkipAddingHostMetadata(t *testing.T) { cases := []struct { title string - event beat.Event + event *beat.Event expectedSkip bool }{ { "event only with host.name", - beat.Event{ + &beat.Event{ Fields: mapstr.M{ "host": mapstr.M{ "name": hostName, @@ -400,7 +419,7 @@ func TestSkipAddingHostMetadata(t *testing.T) { }, { "event only with host.id", - beat.Event{ + &beat.Event{ Fields: mapstr.M{ "host": mapstr.M{ "id": hostID, @@ -411,7 +430,7 @@ func TestSkipAddingHostMetadata(t *testing.T) { }, { "event with host.name and host.id", - beat.Event{ + &beat.Event{ Fields: mapstr.M{ "host": mapstr.M{ "name": hostName, @@ -423,14 +442,14 @@ func TestSkipAddingHostMetadata(t *testing.T) { }, { "event without host field", - beat.Event{ + &beat.Event{ Fields: mapstr.M{}, }, false, }, { "event with field type map[string]string hostID", - beat.Event{ + &beat.Event{ Fields: mapstr.M{ "host": hostIDMap, }, @@ -439,7 +458,7 @@ func TestSkipAddingHostMetadata(t *testing.T) { }, { "event with field type map[string]string host name", - beat.Event{ + &beat.Event{ Fields: mapstr.M{ "host": hostNameMap, }, @@ -448,7 +467,7 @@ func TestSkipAddingHostMetadata(t *testing.T) { }, { "event with field type map[string]string host ID and name", - beat.Event{ + &beat.Event{ Fields: mapstr.M{ "host": hostIDNameMap, }, @@ -457,7 +476,7 @@ func TestSkipAddingHostMetadata(t *testing.T) { }, { "event with field type string", - beat.Event{ + &beat.Event{ Fields: mapstr.M{ "host": "string", }, @@ -468,7 +487,8 @@ func TestSkipAddingHostMetadata(t *testing.T) { for _, c := range cases { t.Run(c.title, func(t *testing.T) { - skip := skipAddingHostMetadata(&c.event) + ed := beat.NewEventEditor(c.event) + skip := skipAddingHostMetadata(ed) assert.Equal(t, c.expectedSkip, skip) }) } @@ -516,11 +536,15 @@ func TestFQDNEventSync(t *testing.T) { for i := 0; i < 10; i++ { checkWait.Add(1) go func() { - resp, err := p.Run(&beat.Event{ + event := &beat.Event{ Fields: mapstr.M{}, - }) + } + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) require.NoError(t, err) - name, err := resp.Fields.GetValue("host.name") + require.False(t, dropped) + ed.Apply() + name, err := event.GetValue("host.name") require.NoError(t, err) require.Equal(t, "foo.bar.baz", name) checkWait.Done() @@ -594,10 +618,12 @@ func TestFQDNLookup(t *testing.T) { Fields: mapstr.M{}, Timestamp: time.Now(), } - newEvent, err := p.Run(event) + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) require.NoError(t, err) - - v, err := newEvent.GetValue("host.name") + require.False(t, dropped) + ed.Apply() + v, err := event.GetValue("host.name") require.NoError(t, err) require.Equal(t, test.expectedHostName, v) }) diff --git a/libbeat/processors/add_id/add_id.go b/libbeat/processors/add_id/add_id.go index 9374a93e1661..062891a5016c 100644 --- a/libbeat/processors/add_id/add_id.go +++ b/libbeat/processors/add_id/add_id.go @@ -60,14 +60,14 @@ func New(cfg *conf.C) (beat.Processor, error) { } // Run enriches the given event with an ID -func (p *addID) Run(event *beat.Event) (*beat.Event, error) { +func (p *addID) Run(event *beat.EventEditor) (dropped bool, err error) { id := p.gen.NextID() if _, err := event.PutValue(p.config.TargetField, id); err != nil { - return nil, makeErrComputeID(err) + return true, makeErrComputeID(err) } - return event, nil + return false, nil } func (p *addID) String() string { diff --git a/libbeat/processors/add_id/add_id_test.go b/libbeat/processors/add_id/add_id_test.go index 18effb85205b..3891035f1444 100644 --- a/libbeat/processors/add_id/add_id_test.go +++ b/libbeat/processors/add_id/add_id_test.go @@ -33,11 +33,13 @@ func TestDefaultTargetField(t *testing.T) { assert.NoError(t, err) testEvent := &beat.Event{} - - newEvent, err := p.Run(testEvent) + ed := beat.NewEventEditor(testEvent) + dropped, err := p.Run(ed) assert.NoError(t, err) + assert.False(t, dropped) + ed.Apply() - v, err := newEvent.GetValue("@metadata._id") + v, err := testEvent.GetValue("@metadata._id") assert.NoError(t, err) assert.NotEmpty(t, v) } @@ -49,19 +51,20 @@ func TestNonDefaultTargetField(t *testing.T) { p, err := New(cfg) assert.NoError(t, err) - testEvent := &beat.Event{ - Fields: mapstr.M{}, - } - - newEvent, err := p.Run(testEvent) + testEvent := &beat.Event{} + ed := beat.NewEventEditor(testEvent) + dropped, err := p.Run(ed) assert.NoError(t, err) + assert.False(t, dropped) + ed.Apply() - v, err := newEvent.GetValue("foo") + v, err := testEvent.GetValue("foo") assert.NoError(t, err) assert.NotEmpty(t, v) - v, err = newEvent.GetValue("@metadata._id") - assert.NoError(t, err) + v, err = testEvent.GetValue("@metadata._id") + assert.Error(t, err) + assert.ErrorIs(t, err, mapstr.ErrKeyNotFound) assert.Empty(t, v) } @@ -72,17 +75,18 @@ func TestNonDefaultMetadataTarget(t *testing.T) { p, err := New(cfg) assert.NoError(t, err) - testEvent := &beat.Event{ - Meta: mapstr.M{}, - } - - newEvent, err := p.Run(testEvent) + testEvent := &beat.Event{} + ed := beat.NewEventEditor(testEvent) + dropped, err := p.Run(ed) assert.NoError(t, err) + assert.False(t, dropped) + ed.Apply() - v, err := newEvent.Meta.GetValue("foo") + v, err := testEvent.GetValue("@metadata.foo") assert.NoError(t, err) assert.NotEmpty(t, v) - v, err = newEvent.GetValue("@metadata._id") + v, err = testEvent.GetValue("@metadata._id") assert.Error(t, err) + assert.ErrorIs(t, err, mapstr.ErrKeyNotFound) } diff --git a/libbeat/processors/conditionals.go b/libbeat/processors/conditionals.go index e5e65660c8ef..035ddb9fc151 100644 --- a/libbeat/processors/conditionals.go +++ b/libbeat/processors/conditionals.go @@ -77,9 +77,9 @@ func NewConditionRule( } // Run executes this WhenProcessor. -func (r *WhenProcessor) Run(event *beat.Event) (*beat.Event, error) { +func (r *WhenProcessor) Run(event *beat.EventEditor) (dropped bool, err error) { if !(r.condition).Check(event) { - return event, nil + return false, nil } return r.p.Run(event) } @@ -162,13 +162,13 @@ func NewIfElseThenProcessor(cfg *config.C) (*IfThenElseProcessor, error) { // Run checks the if condition and executes the processors attached to the // then statement or the else statement based on the condition. -func (p *IfThenElseProcessor) Run(event *beat.Event) (*beat.Event, error) { +func (p *IfThenElseProcessor) Run(event *beat.EventEditor) (dropped bool, err error) { if p.cond.Check(event) { return p.then.Run(event) } else if p.els != nil { return p.els.Run(event) } - return event, nil + return false, nil } func (p *IfThenElseProcessor) String() string { diff --git a/libbeat/processors/conditionals_test.go b/libbeat/processors/conditionals_test.go index 11544625a2e3..d0a9dd2eb858 100644 --- a/libbeat/processors/conditionals_test.go +++ b/libbeat/processors/conditionals_test.go @@ -33,9 +33,9 @@ type countFilter struct { N int } -func (c *countFilter) Run(e *beat.Event) (*beat.Event, error) { +func (c *countFilter) Run(e *beat.EventEditor) (dropped bool, err error) { c.N++ - return e, nil + return false, nil } func (c *countFilter) String() string { return "count" } @@ -100,10 +100,10 @@ func TestWhenProcessor(t *testing.T) { } for _, fields := range test.events { - event := &beat.Event{ + event := beat.NewEventEditor(&beat.Event{ Timestamp: time.Now(), Fields: fields, - } + }) _, err := filter.Run(event) if err != nil { t.Error(err) @@ -149,11 +149,13 @@ func testProcessors(t *testing.T, cases map[string]testCase) { t.Fatal(err) } - result, err := processor.Run(&beat.Event{Fields: test.event.Clone()}) + ed := beat.NewEventEditor(&beat.Event{Fields: test.event}) + dropped, err := processor.Run(ed) if err != nil { t.Fatal(err) } - assert.Equal(t, test.want, result.Fields) + assert.False(t, dropped) + assert.Equal(t, test.want, ed.Fields()) }) } } diff --git a/libbeat/processors/namespace_test.go b/libbeat/processors/namespace_test.go index 07c62328c2c1..a420480c4133 100644 --- a/libbeat/processors/namespace_test.go +++ b/libbeat/processors/namespace_test.go @@ -29,7 +29,7 @@ import ( type testFilterRule struct { str func() string - run func(*beat.Event) (*beat.Event, error) + run func(*beat.EventEditor) (dropped bool, err error) } func TestNamespace(t *testing.T) { @@ -132,9 +132,9 @@ func (r *testFilterRule) String() string { return r.str() } -func (r *testFilterRule) Run(evt *beat.Event) (*beat.Event, error) { +func (r *testFilterRule) Run(evt *beat.EventEditor) (dropped bool, err error) { if r.run == nil { - return evt, nil + return false, nil } return r.Run(evt) } diff --git a/libbeat/processors/processor.go b/libbeat/processors/processor.go index d1f84aee33e3..dafe3c00a5f7 100644 --- a/libbeat/processors/processor.go +++ b/libbeat/processors/processor.go @@ -158,22 +158,31 @@ func (procs *Processors) Close() error { return errs.Err() } -// Run executes the all processors serially and returns the event and possibly -// an error. If the event has been dropped (canceled) by a processor in the -// list then a nil event is returned. -func (procs *Processors) Run(event *beat.Event) (*beat.Event, error) { - var err error +// Run executes all processors from the list serially and applies changes of each +// successfully run processor to the event through the event editor. +// If one of the processors drops the event, this processor also returns `dropped=true` and no error. +func (procs *Processors) Run(event *beat.EventEditor) (dropped bool, err error) { for _, p := range procs.List { - event, err = p.Run(event) - if err != nil { - return event, fmt.Errorf("failed applying processor %v: %w", p, err) + dropped, err = p.Run(event) + if dropped { + return dropped, err } - if event == nil { - // Drop. - return nil, nil + if err != nil { + // if the processor returns an error + // most-likely the changes are incomplete and we don't want + // them to be applied to the event. + event.Reset() + // however, we want to document the error if the processor returned an `EventError` + ee, isEventError := err.(beat.EventError) + if isEventError { + event.AddError(ee) + event.Apply() + } + return dropped, fmt.Errorf("failed applying processor %s: %w", p.String(), err) } + event.Apply() } - return event, nil + return false, nil } func (procs Processors) String() string { diff --git a/libbeat/processors/processor_test.go b/libbeat/processors/processor_test.go index b9370eb4489a..223f7a6b20c4 100644 --- a/libbeat/processors/processor_test.go +++ b/libbeat/processors/processor_test.go @@ -128,9 +128,9 @@ func TestIncludeFields(t *testing.T) { "type": "process", }, } - - processedEvent, err := processors.Run(event) - if err != nil { + ed := beat.NewEventEditor(event) + dropped, err := processors.Run(ed) + if err != nil || dropped { t.Fatal(err) } @@ -148,8 +148,8 @@ func TestIncludeFields(t *testing.T) { }, "type": "process", } - - assert.Equal(t, expectedEvent, processedEvent.Fields) + ed.Apply() + assert.Equal(t, expectedEvent, event.Fields) } func TestIncludeFields1(t *testing.T) { @@ -198,13 +198,14 @@ func TestIncludeFields1(t *testing.T) { }, } - processedEvent, _ := processors.Run(event) + ed := beat.NewEventEditor(event) + _, _ = processors.Run(ed) expectedEvent := mapstr.M{ "type": "process", } - - assert.Equal(t, expectedEvent, processedEvent.Fields) + ed.Apply() + assert.Equal(t, expectedEvent, event.Fields) } func TestDropFields(t *testing.T) { @@ -216,7 +217,7 @@ func TestDropFields(t *testing.T) { "beat.hostname": "mar", }, }, - "fields": []string{"proc.cpu.start_time", "mem", "proc.cmdline", "beat", "dd"}, + "fields": []string{"proc.cpu.start_time", "mem", "proc.cmdline", "beat"}, }, }, } @@ -251,7 +252,10 @@ func TestDropFields(t *testing.T) { }, } - processedEvent, _ := processors.Run(event) + ed := beat.NewEventEditor(event) + dropped, err := processors.Run(ed) + assert.NoError(t, err) + assert.False(t, dropped) expectedEvent := mapstr.M{ "proc": mapstr.M{ @@ -265,7 +269,7 @@ func TestDropFields(t *testing.T) { "type": "process", } - assert.Equal(t, expectedEvent, processedEvent.Fields) + assert.Equal(t, expectedEvent, event.Fields) } func TestMultipleIncludeFields(t *testing.T) { @@ -294,7 +298,6 @@ func TestMultipleIncludeFields(t *testing.T) { event1 := &beat.Event{ Timestamp: time.Now(), Fields: mapstr.M{ - "@timestamp": "2016-01-24T18:35:19.308Z", "beat": mapstr.M{ "hostname": "mar", "name": "my-shipper-1", @@ -358,11 +361,13 @@ func TestMultipleIncludeFields(t *testing.T) { "type": "process", } - actual1, _ := processors.Run(event1) - actual2, _ := processors.Run(event2) + ed1 := beat.NewEventEditor(event1) + ed2 := beat.NewEventEditor(event2) + _, _ = processors.Run(ed1) + _, _ = processors.Run(ed2) - assert.Equal(t, expected1, actual1.Fields) - assert.Equal(t, expected2, actual2.Fields) + assert.Equal(t, expected1, event1.Fields) + assert.Equal(t, expected2, event2.Fields) } func TestDropEvent(t *testing.T) { @@ -412,9 +417,10 @@ func TestDropEvent(t *testing.T) { }, } - processedEvent, _ := processors.Run(event) + ed := beat.NewEventEditor(event) + dropped, _ := processors.Run(ed) - assert.Nil(t, processedEvent) + assert.True(t, dropped) } func TestEmptyCondition(t *testing.T) { @@ -456,9 +462,10 @@ func TestEmptyCondition(t *testing.T) { }, } - processedEvent, _ := processors.Run(event) + ed := beat.NewEventEditor(event) + dropped, _ := processors.Run(ed) - assert.Nil(t, processedEvent) + assert.True(t, dropped) } func TestBadCondition(t *testing.T) { @@ -523,7 +530,8 @@ func TestDropMissingFields(t *testing.T) { yml := []map[string]interface{}{ { "drop_fields": map[string]interface{}{ - "fields": []string{"foo.bar", "proc.cpu", "proc.sss", "beat", "mem"}, + "ignore_missing": true, + "fields": []string{"foo.bar", "proc.cpu", "proc.sss", "beat", "mem"}, }, }, } @@ -558,7 +566,10 @@ func TestDropMissingFields(t *testing.T) { }, } - processedEvent, _ := processors.Run(event) + ed := beat.NewEventEditor(event) + dropped, err := processors.Run(ed) + assert.False(t, dropped) + assert.NoError(t, err) expectedEvent := mapstr.M{ "proc": mapstr.M{ @@ -567,7 +578,7 @@ func TestDropMissingFields(t *testing.T) { "type": "process", } - assert.Equal(t, expectedEvent, processedEvent.Fields) + assert.Equal(t, expectedEvent, event.Fields) } const ( @@ -578,10 +589,10 @@ const ( func BenchmarkProcessorsRun(b *testing.B) { processors := processors.NewList(nil) key1 := "added.field" - proc1 := actions.NewAddFields(mapstr.M{key1: "first"}, true, true) + proc1 := actions.NewAddFields(mapstr.M{key1: "first"}, true) processors.AddProcessor(proc1) key2 := "field-0.field-0" - proc2 := actions.NewAddFields(mapstr.M{key2: "second"}, true, true) + proc2 := actions.NewAddFields(mapstr.M{key2: "second"}, true) processors.AddProcessor(proc2) event := &beat.Event{ @@ -593,24 +604,26 @@ func BenchmarkProcessorsRun(b *testing.B) { generateFields(b, event.Meta, 100, 2) generateFields(b, event.Fields, 100, 2) + ed := beat.NewEventEditor(event) + var ( - processed *beat.Event - err error + dropped bool + err error ) b.Run("processors.Run", func(b *testing.B) { for i := 0; i < b.N; i++ { - processed, err = processors.Run(event) + dropped, err = processors.Run(ed) require.NoError(b, err) - require.NotNil(b, processed) + require.False(b, dropped) } }) - added, err := processed.GetValue(key1) + added, err := ed.GetValue(key1) require.NoError(b, err) require.Equal(b, "first", added) - added, err = processed.GetValue(key2) + added, err = ed.GetValue(key2) require.NoError(b, err) require.Equal(b, "second", added) } diff --git a/libbeat/processors/safe_processor.go b/libbeat/processors/safe_processor.go index a0bbf5824d07..2f9f9f0c2996 100644 --- a/libbeat/processors/safe_processor.go +++ b/libbeat/processors/safe_processor.go @@ -34,9 +34,9 @@ type SafeProcessor struct { } // Run allows to run processor only when `Close` was not called prior -func (p *SafeProcessor) Run(event *beat.Event) (*beat.Event, error) { +func (p *SafeProcessor) Run(event *beat.EventEditor) (dropped bool, err error) { if atomic.LoadUint32(&p.closed) == 1 { - return nil, ErrClosed + return true, ErrClosed } return p.Processor.Run(event) } diff --git a/libbeat/processors/safe_processor_test.go b/libbeat/processors/safe_processor_test.go index 98ad4e7dba86..adbe8aeb648b 100644 --- a/libbeat/processors/safe_processor_test.go +++ b/libbeat/processors/safe_processor_test.go @@ -32,9 +32,9 @@ type mockProcessor struct { runCount int } -func (p *mockProcessor) Run(event *beat.Event) (*beat.Event, error) { +func (p *mockProcessor) Run(event *beat.EventEditor) (bool, error) { p.runCount++ - return mockEvent, nil + return false, nil } func (p *mockProcessor) String() string { @@ -100,12 +100,13 @@ func TestSafeProcessor(t *testing.T) { t.Run("propagates Run to a processor", func(t *testing.T) { require.Equal(t, 0, p.runCount) - e, err := sp.Run(nil) + dropped, err := sp.Run(nil) require.NoError(t, err) - require.Equal(t, e, mockEvent) - e, err = sp.Run(nil) + require.False(t, dropped) + + dropped, err = sp.Run(nil) require.NoError(t, err) - require.Equal(t, e, mockEvent) + require.False(t, dropped) require.Equal(t, 2, p.runCount) }) @@ -123,8 +124,8 @@ func TestSafeProcessor(t *testing.T) { t.Run("does not propagate Run when closed", func(t *testing.T) { require.Equal(t, 2, p.runCount) // still 2 from the previous test case - e, err := sp.Run(nil) - require.Nil(t, e) + dropped, err := sp.Run(nil) + require.True(t, dropped) require.ErrorIs(t, err, ErrClosed) require.Equal(t, 2, p.runCount) }) diff --git a/libbeat/processors/script/javascript/beatevent_v0.go b/libbeat/processors/script/javascript/beatevent_v0.go index 82b011fb991e..b50d98be8f76 100644 --- a/libbeat/processors/script/javascript/beatevent_v0.go +++ b/libbeat/processors/script/javascript/beatevent_v0.go @@ -36,7 +36,7 @@ import ( type beatEventV0 struct { vm *goja.Runtime obj *goja.Object - inner *beat.Event + inner *beat.EventEditor cancelled bool } @@ -73,7 +73,7 @@ func newBeatEventV0Constructor(s Session) func(call goja.ConstructorCall) *goja. obj: call.This, } evt.init() - evt.reset(&beat.Event{Fields: fields}) + evt.reset(beat.NewEventEditor(&beat.Event{Fields: fields})) return nil } } @@ -89,16 +89,15 @@ func (e *beatEventV0) init() { } // reset the event so that it can be reused to wrap another event. -func (e *beatEventV0) reset(b *beat.Event) error { - e.inner = b +func (e *beatEventV0) reset(ed *beat.EventEditor) error { + e.inner = ed e.cancelled = false e.obj.Set("_private", e) - e.obj.Set("fields", e.vm.ToValue(e.inner.Fields)) return nil } -// Wrapped returns the wrapped beat.Event. -func (e *beatEventV0) Wrapped() *beat.Event { +// Wrapped returns the wrapped beat.EventEditor. +func (e *beatEventV0) Wrapped() *beat.EventEditor { return e.inner } @@ -116,8 +115,8 @@ func (e *beatEventV0) JSObject() goja.Value { func (e *beatEventV0) get(call goja.FunctionCall) goja.Value { a0 := call.Argument(0) if goja.IsUndefined(a0) { - // event.Get() is the same as event.fields (but slower). - return e.vm.ToValue(e.inner.Fields) + // event.Get() returns the whole fields map, it's slow and expensive + return e.vm.ToValue(e.inner.Fields()) } v, err := e.inner.GetValue(a0.String()) @@ -234,7 +233,7 @@ func (e *beatEventV0) tag(call goja.FunctionCall) goja.Value { tag := call.Argument(0).String() - if err := appendString(e.inner.Fields, "tags", tag, true); err != nil { + if err := appendString(e.inner, "tags", tag, true); err != nil { panic(err) } return goja.Undefined() @@ -255,24 +254,24 @@ func (e *beatEventV0) appendTo(call goja.FunctionCall) goja.Value { field := call.Argument(0).String() value := call.Argument(1).String() - if err := appendString(e.inner.Fields, field, value, false); err != nil { + if err := appendString(e.inner, field, value, false); err != nil { panic(err) } return goja.Undefined() } -func appendString(m mapstr.M, field, value string, alwaysArray bool) error { - list, _ := m.GetValue(field) +func appendString(ed *beat.EventEditor, field, value string, alwaysArray bool) error { + list, _ := ed.GetValue(field) switch v := list.(type) { case nil: if alwaysArray { - m.Put(field, []string{value}) + ed.PutValue(field, []string{value}) } else { - m.Put(field, value) + ed.PutValue(field, value) } case string: if value != v { - m.Put(field, []string{v, value}) + ed.PutValue(field, []string{v, value}) } case []string: for _, existingTag := range v { @@ -281,7 +280,7 @@ func appendString(m mapstr.M, field, value string, alwaysArray bool) error { return nil } } - m.Put(field, append(v, value)) + ed.PutValue(field, append(v, value)) case []interface{}: for _, existingTag := range v { if value == existingTag { @@ -289,7 +288,7 @@ func appendString(m mapstr.M, field, value string, alwaysArray bool) error { return nil } } - m.Put(field, append(v, value)) + ed.PutValue(field, append(v, value)) default: return fmt.Errorf("unexpected type %T found for %v field", list, field) } diff --git a/libbeat/processors/script/javascript/beatevent_v0_test.go b/libbeat/processors/script/javascript/beatevent_v0_test.go index 9d29a37b1b0f..fb26375cfdf9 100644 --- a/libbeat/processors/script/javascript/beatevent_v0_test.go +++ b/libbeat/processors/script/javascript/beatevent_v0_test.go @@ -26,7 +26,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/beats/v7/libbeat/beat/events" "github.com/elastic/beats/v7/libbeat/tests/resources" "github.com/elastic/elastic-agent-libs/mapstr" "github.com/elastic/elastic-agent-libs/monitoring" @@ -40,24 +39,17 @@ const ( type testCase struct { name string source string - assert func(t testing.TB, evt *beat.Event, err error) + assert func(t testing.TB, evt *beat.EventEditor, dropped bool, err error) } var eventV0Tests = []testCase{ { name: "Put", source: `evt.Put("hello", "world");`, - assert: func(t testing.TB, evt *beat.Event, err error) { - v, _ := evt.GetValue("hello") - assert.Equal(t, "world", v) - }, - }, - { - name: "Object Put Key", - source: `evt.fields["hello"] = "world";`, - assert: func(t testing.TB, evt *beat.Event, err error) { + assert: func(t testing.TB, evt *beat.EventEditor, dropped bool, err error) { v, _ := evt.GetValue("hello") assert.Equal(t, "world", v) + assert.False(t, dropped) }, }, { @@ -87,29 +79,22 @@ var eventV0Tests = []testCase{ throw "failed to get IP"; }`, }, - { - name: "fields get key", - source: ` - var ip = evt.fields.source.ip; - - if ("192.0.2.1" !== ip) { - throw "failed to get IP"; - }`, - }, { name: "Delete", source: `if (!evt.Delete("source.ip")) { throw "delete failed"; }`, - assert: func(t testing.TB, evt *beat.Event, err error) { + assert: func(t testing.TB, evt *beat.EventEditor, dropped bool, err error) { ip, _ := evt.GetValue("source.ip") assert.Nil(t, ip) + assert.False(t, dropped) }, }, { name: "Rename", source: `if (!evt.Rename("source", "destination")) { throw "rename failed"; }`, - assert: func(t testing.TB, evt *beat.Event, err error) { + assert: func(t testing.TB, evt *beat.EventEditor, dropped bool, err error) { ip, _ := evt.GetValue("destination.ip") assert.Equal(t, "192.0.2.1", ip) + assert.False(t, dropped) }, }, { @@ -121,48 +106,58 @@ var eventV0Tests = []testCase{ { name: "Put @metadata", source: `evt.Put("@metadata.foo", "bar");`, - assert: func(t testing.TB, evt *beat.Event, err error) { - assert.Equal(t, "bar", evt.Meta["foo"]) + assert: func(t testing.TB, evt *beat.EventEditor, dropped bool, err error) { + meta, _ := evt.GetValue("@metadata.foo") + assert.Equal(t, "bar", meta) + assert.False(t, dropped) }, }, { name: "Delete @metadata", source: `evt.Delete("@metadata.pipeline");`, - assert: func(t testing.TB, evt *beat.Event, err error) { - assert.Nil(t, evt.Meta[events.FieldMetaPipeline]) + assert: func(t testing.TB, evt *beat.EventEditor, dropped bool, err error) { + val, getErr := evt.GetValue("@metadata.pipeline") + assert.Nil(t, val) + assert.NoError(t, err) + assert.Error(t, getErr) + assert.ErrorIs(t, getErr, mapstr.ErrKeyNotFound) + assert.False(t, dropped) }, }, { name: "Cancel", source: `evt.Cancel();`, - assert: func(t testing.TB, evt *beat.Event, err error) { + assert: func(t testing.TB, evt *beat.EventEditor, dropped bool, err error) { assert.NoError(t, err) - assert.Nil(t, evt) + assert.True(t, dropped) }, }, { name: "Tag", source: `evt.Tag("foo"); evt.Tag("bar"); evt.Tag("foo");`, - assert: func(t testing.TB, evt *beat.Event, err error) { - if assert.NoError(t, err) { - assert.Equal(t, []string{"foo", "bar"}, evt.Fields["tags"]) - } + assert: func(t testing.TB, evt *beat.EventEditor, dropped bool, err error) { + assert.NoError(t, err) + val, getErr := evt.GetValue(mapstr.TagsKey) + assert.NoError(t, getErr) + assert.Equal(t, []string{"foo", "bar"}, val) + assert.False(t, dropped) }, }, { name: "AppendTo", source: `evt.AppendTo("source.ip", "10.0.0.1");`, - assert: func(t testing.TB, evt *beat.Event, err error) { + assert: func(t testing.TB, evt *beat.EventEditor, dropped bool, err error) { if assert.NoError(t, err) { srcIP, _ := evt.GetValue("source.ip") assert.Equal(t, []string{"192.0.2.1", "10.0.0.1"}, srcIP) } + assert.False(t, dropped) }, }, } -func testEvent() *beat.Event { - return &beat.Event{ +func testEvent() *beat.EventEditor { + return beat.NewEventEditor(&beat.Event{ Meta: mapstr.M{ "pipeline": "beat-1.2.3-module", }, @@ -171,7 +166,7 @@ func testEvent() *beat.Event { "ip": "192.0.2.1", }, }, - } + }) } func TestBeatEventV0(t *testing.T) { @@ -184,9 +179,10 @@ func TestBeatEventV0(t *testing.T) { t.Fatal(err) } - evt, err := p.Run(testEvent()) + evt := testEvent() + dropped, err := p.Run(evt) if tc.assert != nil { - tc.assert(t, evt, err) + tc.assert(t, evt, dropped, err) } else { assert.NoError(t, err) assert.NotNil(t, evt) diff --git a/libbeat/processors/script/javascript/javascript.go b/libbeat/processors/script/javascript/javascript.go index 527bd03e92ce..8937dc46be0c 100644 --- a/libbeat/processors/script/javascript/javascript.go +++ b/libbeat/processors/script/javascript/javascript.go @@ -164,31 +164,28 @@ func annotateError(id string, err error) error { // Run executes the processor on the given it event. It invokes the // process function defined in the Javascript source. -func (p *jsProcessor) Run(event *beat.Event) (*beat.Event, error) { +func (p *jsProcessor) Run(event *beat.EventEditor) (dropped bool, err error) { s := p.sessionPool.Get() defer p.sessionPool.Put(s) - var rtn *beat.Event - var err error - if p.stats == nil { - rtn, err = s.runProcessFunc(event) + dropped, err = s.runProcessFunc(event) } else { - rtn, err = p.runWithStats(s, event) + dropped, err = p.runWithStats(s, event) } - return rtn, annotateError(p.Tag, err) + return dropped, annotateError(p.Tag, err) } -func (p *jsProcessor) runWithStats(s *session, event *beat.Event) (*beat.Event, error) { +func (p *jsProcessor) runWithStats(s *session, event *beat.EventEditor) (dropped bool, err error) { start := time.Now() - event, err := s.runProcessFunc(event) + dropped, err = s.runProcessFunc(event) elapsed := time.Since(start) p.stats.processTime.Update(int64(elapsed)) if err != nil { p.stats.exceptions.Inc() } - return event, err + return dropped, err } func (p *jsProcessor) String() string { diff --git a/libbeat/processors/script/javascript/module/processor/chain.go b/libbeat/processors/script/javascript/module/processor/chain.go index 04982947b302..0057a9809551 100644 --- a/libbeat/processors/script/javascript/module/processor/chain.go +++ b/libbeat/processors/script/javascript/module/processor/chain.go @@ -169,11 +169,11 @@ func newNativeProcessor(constructor processors.Constructor, call gojaCall) (proc } func (p *nativeProcessor) run(event javascript.Event) error { - out, err := p.Processor.Run(event.Wrapped()) + dropped, err := p.Processor.Run(event.Wrapped()) if err != nil { return err } - if out == nil { + if dropped { event.Cancel() } return nil diff --git a/libbeat/processors/script/javascript/session.go b/libbeat/processors/script/javascript/session.go index 337d2eae7997..f7716838d938 100644 --- a/libbeat/processors/script/javascript/session.go +++ b/libbeat/processors/script/javascript/session.go @@ -60,14 +60,14 @@ type Event interface { // Wrapped returns the underlying beat.Event being wrapped. The wrapped // event is replaced each time a new event is processed. - Wrapped() *beat.Event + Wrapped() *beat.EventEditor // JSObject returns the Value that represents this object within the // runtime. JSObject() goja.Value // reset replaces the inner beat.Event and resets the state. - reset(*beat.Event) error + reset(*beat.EventEditor) error } // session is a javascript runtime environment used throughout the life of @@ -192,7 +192,7 @@ func (s *session) executeTestFunction() error { } // setEvent replaces the beat event handle present in the runtime. -func (s *session) setEvent(b *beat.Event) error { +func (s *session) setEvent(b *beat.EventEditor) error { if s.evt == nil { var err error s.evt, err = s.makeEvent(s) @@ -205,28 +205,28 @@ func (s *session) setEvent(b *beat.Event) error { } // runProcessFunc executes process() from the JS script. -func (s *session) runProcessFunc(b *beat.Event) (out *beat.Event, err error) { +func (s *session) runProcessFunc(b *beat.EventEditor) (dropped bool, err error) { defer func() { if r := recover(); r != nil { s.log.Errorw("The javascript processor caused an unexpected panic "+ "while processing an event. Recovering, but please report this.", - "event", mapstr.M{"original": b.Fields.String()}, + "event", b.String(), "panic", r, zap.Stack("stack")) if !s.evt.IsCancelled() { - out = b + b.Reset() } err = fmt.Errorf("unexpected panic in javascript processor: %v", r) if s.tagOnException != "" { - mapstr.AddTags(b.Fields, []string{s.tagOnException}) + b.AddTags(s.tagOnException) } - appendString(b.Fields, "error.message", err.Error(), false) + appendString(b, "error.message", err.Error(), false) } }() if err = s.setEvent(b); err != nil { // Always return the event even if there was an error. - return b, err + return false, err } // Interrupt the JS code if execution exceeds timeout. @@ -239,16 +239,16 @@ func (s *session) runProcessFunc(b *beat.Event) (out *beat.Event, err error) { if _, err = s.processFunc(goja.Undefined(), s.evt.JSObject()); err != nil { if s.tagOnException != "" { - mapstr.AddTags(b.Fields, []string{s.tagOnException}) + b.AddTags(s.tagOnException) } - appendString(b.Fields, "error.message", err.Error(), false) - return b, fmt.Errorf("failed in process function: %w", err) + appendString(b, "error.message", err.Error(), false) + return dropped, fmt.Errorf("failed in process function: %w", err) } if s.evt.IsCancelled() { - return nil, nil + return true, nil } - return b, nil + return false, nil } // Runtime returns the Javascript runtime used for this session. diff --git a/libbeat/processors/script/javascript/session_test.go b/libbeat/processors/script/javascript/session_test.go index 03d8a4236333..0900b86ca001 100644 --- a/libbeat/processors/script/javascript/session_test.go +++ b/libbeat/processors/script/javascript/session_test.go @@ -41,10 +41,11 @@ func TestSessionTagOnException(t *testing.T) { t.Fatal(err) } - evt, err := p.Run(testEvent()) + evt := testEvent() + _, err = p.Run(evt) assert.Error(t, err) - tags, _ := evt.GetValue("tags") + tags, _ := evt.GetValue(mapstr.TagsKey) assert.Equal(t, []string{"_js_exception"}, tags) } @@ -109,7 +110,7 @@ func TestSessionTestFunction(t *testing.T) { function test() { var event = process(new Event({"hello": "earth"})); - if (event.fields.hello !== "world") { + if (event.Get("hello") !== "world") { throw "invalid hello world"; } } @@ -149,8 +150,8 @@ func TestSessionTimeout(t *testing.T) { logp.TestingSetup() const runawayLoop = ` - while (!evt.fields.stop) { - evt.Put("hello", "world"); + while (!evt.Get("stop")) { + evt.Put("hello", "world"); } ` @@ -163,14 +164,14 @@ func TestSessionTimeout(t *testing.T) { t.Fatal(err) } - evt := &beat.Event{ + evt := beat.NewEventEditor(&beat.Event{ Fields: mapstr.M{ "stop": false, }, - } + }) // Execute and expect a timeout. - evt, err = p.Run(evt) + _, err = p.Run(evt) if assert.Error(t, err) { assert.Contains(t, err.Error(), timeoutError) @@ -189,7 +190,7 @@ func TestSessionTimeout(t *testing.T) { func TestSessionParallel(t *testing.T) { const script = ` - evt.Put("host.name", "workstation"); + evt.Put("host.name", "workstation"); ` p, err := NewFromConfig(Config{ @@ -210,11 +211,11 @@ func TestSessionParallel(t *testing.T) { go func() { defer wg.Done() for ctx.Err() == nil { - evt := &beat.Event{ + evt := beat.NewEventEditor(&beat.Event{ Fields: mapstr.M{ "host": mapstr.M{"name": "computer"}, }, - } + }) _, err := p.Run(evt) assert.NoError(t, err) }