diff --git a/libbeat/beat/event.go b/libbeat/beat/event.go index 6db34702b96d..e748c94a8116 100644 --- a/libbeat/beat/event.go +++ b/libbeat/beat/event.go @@ -38,9 +38,11 @@ var ( const FlagField = "log.flags" const ( - timestampFieldKey = "@timestamp" - metadataFieldKey = "@metadata" - metadataKeyPrefix = metadataFieldKey + "." + TimestampFieldKey = "@timestamp" + MetadataFieldKey = "@metadata" + ErrorFieldKey = "error" + TypeFieldKey = "type" + metadataKeyPrefix = MetadataFieldKey + "." metadataKeyOffset = len(metadataKeyPrefix) ) @@ -59,9 +61,9 @@ type Event struct { var ( ErrValueNotTimestamp = errors.New("value is not a timestamp") ErrValueNotMapStr = errors.New("value is not `mapstr.M` or `map[string]interface{}` type") - ErrAlterMetadataKey = fmt.Errorf("deleting/replacing %q key is not supported", metadataFieldKey) - ErrMetadataAccess = fmt.Errorf("accessing %q key directly is not supported, try nested keys", metadataFieldKey) - ErrDeleteTimestamp = fmt.Errorf("deleting %q key is not supported", timestampFieldKey) + ErrAlterMetadataKey = fmt.Errorf("deleting/replacing %q key is not supported", MetadataFieldKey) + ErrMetadataAccess = fmt.Errorf("accessing %q key directly is not supported, try nested keys", MetadataFieldKey) + ErrDeleteTimestamp = fmt.Errorf("deleting %q key is not supported", TimestampFieldKey) ) // SetID overwrites the "id" field in the events metadata. @@ -71,7 +73,7 @@ func (e *Event) SetID(id string) { } func (e *Event) HasKey(key string) (bool, error) { - if key == timestampFieldKey || key == metadataFieldKey { + if key == TimestampFieldKey || key == MetadataFieldKey { return true, nil } @@ -90,10 +92,10 @@ func (e *Event) HasKey(key string) (bool, error) { } func (e *Event) GetValue(key string) (interface{}, error) { - if key == timestampFieldKey { + if key == TimestampFieldKey { return e.Timestamp, nil } - if key == metadataFieldKey { + if key == MetadataFieldKey { return nil, ErrMetadataAccess } @@ -113,15 +115,15 @@ func (e *Event) GetValue(key string) (interface{}, error) { // Clone creates an exact copy of the event // TODO DELETE -func (e *Event) Clone() *Event { - return &Event{ - Timestamp: e.Timestamp, - Meta: e.Meta.Clone(), - Fields: e.Fields.Clone(), - Private: e.Private, - TimeSeries: e.TimeSeries, - } -} +// func (e *Event) Clone() *Event { +// return &Event{ +// Timestamp: e.Timestamp, +// Meta: e.Meta.Clone(), +// Fields: e.Fields.Clone(), +// Private: e.Private, +// TimeSeries: e.TimeSeries, +// } +// } // DeepUpdate recursively copies the key-value pairs from `d` to various properties of the event. // When the key equals `@timestamp` it's set as the `Timestamp` property of the event. @@ -147,10 +149,10 @@ func (e *Event) DeepUpdateNoOverwrite(d mapstr.M) { } func (e *Event) PutValue(key string, v interface{}) (interface{}, error) { - if key == timestampFieldKey { + if key == TimestampFieldKey { return e.setTimestamp(v) } - if key == metadataFieldKey { + if key == MetadataFieldKey { return nil, ErrAlterMetadataKey } @@ -170,10 +172,10 @@ func (e *Event) PutValue(key string, v interface{}) (interface{}, error) { } func (e *Event) Delete(key string) error { - if key == timestampFieldKey { + if key == TimestampFieldKey { return ErrDeleteTimestamp } - if key == metadataFieldKey { + if key == MetadataFieldKey { return ErrAlterMetadataKey } if subKey, ok := e.metadataSubKey(key); ok { @@ -194,7 +196,7 @@ func (e *Event) Delete(key string) error { // error as fields with the corresponding name. func (e *Event) SetErrorWithOption(message string, addErrKey bool, data string, field string) { if addErrKey { - errorField := mapstr.M{"message": message, "type": "json"} + errorField := mapstr.M{"message": message, TypeFieldKey: "json"} if data != "" { errorField["data"] = data } @@ -205,6 +207,16 @@ func (e *Event) SetErrorWithOption(message string, addErrKey bool, data string, } } +// String returns a string representation of the event. +func (e *Event) String() string { + m := mapstr.M{ + TimestampFieldKey: e.Timestamp, + MetadataFieldKey: e.Meta, + } + m.DeepUpdate(e.Fields) + return m.String() +} + func (e *Event) deepUpdate(d mapstr.M, mode updateMode) { if len(d) == 0 { return @@ -212,7 +224,7 @@ func (e *Event) deepUpdate(d mapstr.M, mode updateMode) { // It's supported to update the timestamp using this function. // However, we must handle it separately since it's a separate field of the event. - timestampValue, timestampExists := d[timestampFieldKey] + timestampValue, timestampExists := d[TimestampFieldKey] if timestampExists { if mode == updateModeOverwrite { _, _ = e.setTimestamp(timestampValue) @@ -221,15 +233,15 @@ func (e *Event) deepUpdate(d mapstr.M, mode updateMode) { // Temporary delete it from the update map, // so we can do `e.Fields.DeepUpdate(d)` or // `e.Fields.DeepUpdateNoOverwrite(d)` later - delete(d, timestampFieldKey) + delete(d, TimestampFieldKey) defer func() { - d[timestampFieldKey] = timestampValue + d[TimestampFieldKey] = timestampValue }() } // It's supported to update the metadata using this function. // However, we must handle it separately since it's a separate field of the event. - metaValue, metaExists := d[metadataFieldKey] + metaValue, metaExists := d[MetadataFieldKey] if metaExists { var metaUpdate mapstr.M @@ -255,9 +267,9 @@ func (e *Event) deepUpdate(d mapstr.M, mode updateMode) { // Temporary delete it from the update map, // so we can do `e.Fields.DeepUpdate(d)` or // `e.Fields.DeepUpdateNoOverwrite(d)` later - delete(d, metadataFieldKey) + delete(d, MetadataFieldKey) defer func() { - d[metadataFieldKey] = metaValue + d[MetadataFieldKey] = metaValue }() } diff --git a/libbeat/beat/event_editor.go b/libbeat/beat/event_editor.go index 1d8ece2be194..60a36dd80ede 100644 --- a/libbeat/beat/event_editor.go +++ b/libbeat/beat/event_editor.go @@ -19,6 +19,7 @@ package beat import ( "errors" + "fmt" "strings" "github.com/elastic/elastic-agent-libs/mapstr" @@ -31,50 +32,51 @@ var ( checkoutModeIncludeValues checkoutMode = true ) -// TODO rename to `Event` and use it instead of the actual event struct everywhere -type EventAccessor interface { - // GetValue gets a value from the map. The key can be expressed in dot-notation (e.g. x.y). - // If the key does not exist then `mapstr.ErrKeyNotFound` error is returned. - GetValue(key string) (interface{}, error) - - // PutValue associates the specified value with the specified key. If the event - // previously contained a mapping for the key, the old value is replaced and - // returned. The key can be expressed in dot-notation (e.g. x.y) to put a value - // into a nested map. - // - // If you need insert keys containing dots then you must use bracket notation - // to insert values (e.g. m[key] = value). - PutValue(key string, v interface{}) (interface{}, error) - - // Delete value with the given key. - // The key can be expressed in dot-notation (e.g. x.y) - Delete(key string) error - - // DeepUpdate recursively copies the key-value pairs from `d` to various properties of the event. - // When the key equals `@timestamp` it's set as the `Timestamp` property of the event. - // When the key equals `@metadata` the update is routed into the `Meta` map instead of `Fields` - // The rest of the keys are set to the `Fields` map. - // If the key is present and the value is a map as well, the sub-map will be updated recursively - // via `DeepUpdate`. - // `DeepUpdateNoOverwrite` is a version of this function that does not - // overwrite existing values.p - DeepUpdate(d mapstr.M) - - // DeepUpdateNoOverwrite recursively copies the key-value pairs from `d` to various properties of the event. - // The `@timestamp` update is ignored due to "no overwrite" behavior. - // When the key equals `@metadata` the update is routed into the `Meta` map instead of `Fields`. - // The rest of the keys are set to the `Fields` map. - // If the key is present and the value is a map as well, the sub-map will be updated recursively - // via `DeepUpdateNoOverwrite`. - // `DeepUpdate` is a version of this function that overwrites existing values. - DeepUpdateNoOverwrite(d mapstr.M) +type EventError struct { + Message string + Field string + Data string + Processor string +} + +func (e EventError) Error() string { + var prefix string + if e.Processor != "" { + prefix += fmt.Sprintf("[processor=%s] ", e.Processor) + } + + if e.Field != "" { + prefix += fmt.Sprintf("[field=%q] ", e.Field) + } + + if e.Data != "" { + prefix += fmt.Sprintf("[data=%s] ", e.Data) + } + return prefix + e.Message +} + +func (e EventError) toMap() mapstr.M { + m := mapstr.M{ + "message": e.Message, + } + if e.Data != "" { + m["data"] = e.Data + } + if e.Field != "" { + m["field"] = e.Field + } + if e.Processor != "" { + m["processor"] = e.Processor + } + + return m } // EventEditor is a wrapper that allows to make changes to the wrapped event // preserving states of the original nested maps by cloning them on demand. // // The first time a nested map gets updated it's cloned and, from that moment on, only the copy is modified. -// Once all the changes are collected, users should call `Apply` to copy pending changes to the original event. +// Once all the changes are collected, users can call `Apply` to apply pending changes to the original event. // When the changes get applied the pointers to originally referenced nested maps get replaced with pointers to // modified copies. // @@ -92,6 +94,7 @@ type EventEditor struct { deletions map[string]struct{} } +// NewEventEditor creates a new event editor for the given event. func NewEventEditor(e *Event) *EventEditor { if e == nil { e = &Event{} @@ -101,28 +104,100 @@ func NewEventEditor(e *Event) *EventEditor { } } -// GetValue implements the `EventAccessor` interface. +// Fields returns the current computed state of changes without applying any changes to the original event. +// +// This function is cloning all the fields from the original event. +// Using this function is slow and expensive on memory, try not to use it unless it's absolutely necessary. +func (e *EventEditor) Fields() mapstr.M { + if e.original.Fields == nil { + if e.pending.Fields != nil { + return e.pending.Fields.Clone() + } + return mapstr.M{} + } + fields := e.original.Fields.Clone() + for key := range e.deletions { + _ = fields.Delete(key) + } + if e.pending != nil { + fields.DeepUpdate(e.pending.Fields) + } + return fields +} + +// FlattenKeys returns all flatten keys for the current pending state of the event. +// This includes original undeleted keys and new unapplied keys. +func (e *EventEditor) FlattenKeys() []string { + uniqueKeys := make(map[string]struct{}) + + if e.original.Meta != nil { + for _, key := range *e.original.Meta.FlattenKeys() { + if e.checkDeleted(key) { + continue + } + uniqueKeys[metadataKeyPrefix+key] = struct{}{} + } + } + if e.original.Fields != nil { + for _, key := range *e.original.Fields.FlattenKeys() { + if e.checkDeleted(key) { + continue + } + uniqueKeys[key] = struct{}{} + } + } + + if e.pending != nil { + if e.pending.Meta != nil { + for _, key := range *e.pending.Meta.FlattenKeys() { + uniqueKeys[metadataKeyPrefix+key] = struct{}{} + } + } + if e.pending.Fields != nil { + for _, key := range *e.pending.Fields.FlattenKeys() { + uniqueKeys[key] = struct{}{} + } + } + } + + result := make([]string, 0, len(uniqueKeys)) + for key := range uniqueKeys { + result = append(result, key) + } + return result +} + +// GetValue gets a value from the event. The key can be expressed in dot-notation (e.g. x.y). +// If the key does not exist then `mapstr.ErrKeyNotFound` error is returned. +// +// If the returned value is a nested map this function always returns a copy, +// not the same nested map referenced by the original event. func (e *EventEditor) GetValue(key string) (interface{}, error) { - if key == metadataFieldKey { + if key == "" { + return nil, mapstr.ErrKeyNotFound + } + if key == MetadataFieldKey { return nil, ErrMetadataAccess } - // handle the deletion marks rootKey := e.rootKey(key) - if rootKey == key && e.checkDeleted(key) { - return nil, mapstr.ErrKeyNotFound - } if e.pending != nil { + // if the root key is empty the original event never had this key + if rootKey == "" { + return e.pending.GetValue(key) + } // We try to retrieve from the `pending` event first, // since it should have the most recent data. // // To check if the nested map was checked-out before - // we need to get the root-level first + // we need to get the root-level first. val, err := e.pending.GetValue(rootKey) + // if the key was found, it was either created by PutValue or + // checked out from the original event. if err == nil { if rootKey == key { - // the value might be the end value we're looking for + // the value might be the root-level end value we're looking for return val, nil } else { // otherwise, we need to retrieve from the nested map @@ -143,6 +218,15 @@ func (e *EventEditor) GetValue(key string) (interface{}, error) { } } + // handle the deletion marks but only on the root-level + // the rest should be covered by looking into the pending event + // where we check out all the nested maps. + if rootKey == key && e.checkDeleted(key) { + return nil, mapstr.ErrKeyNotFound + } + + // in case the key was never checked out and it's still + // present only in the original event value, err := e.original.GetValue(key) if err != nil { return nil, err @@ -164,39 +248,58 @@ func (e *EventEditor) GetValue(key string) (interface{}, error) { return e.pending.GetValue(key) } -// PutValue implements the `EventAccessor` interface. +// PutValue associates the specified value with the specified key. If the event +// previously contained a mapping for the key, the old value is replaced and +// returned. The key can be expressed in dot-notation (e.g. x.y) to put a value +// into a nested map. +// +// If the returned previous value is a nested map this function always returns a copy, +// not the same nested map referenced by the original event. +// +// This changed is not applied to the original event until `Apply` is called. func (e *EventEditor) PutValue(key string, v interface{}) (interface{}, error) { - if key == metadataFieldKey { + switch key { + case MetadataFieldKey: return nil, ErrAlterMetadataKey + case TimestampFieldKey: + e.allocatePending() + return e.pending.PutValue(key, v) + default: + e.allocatePending() + // checkout only if the key leads to a nested value + if !e.checkDeleted(key) { + e.checkout(key, checkoutModeIncludeValues) + } + + return e.pending.PutValue(key, v) } - // checkout only if the key leads to a nested value - e.checkout(key, checkoutModeOnlyMaps) - e.allocatePending() - return e.pending.PutValue(key, v) } -// Delete implements the `EventAccessor` interface. +// Delete value with the given key. +// The key can be expressed in dot-notation (e.g. x.y) +// +// This changed is not applied to the original event until `Apply` is called. func (e *EventEditor) Delete(key string) error { - if key == timestampFieldKey { + if key == TimestampFieldKey { return ErrDeleteTimestamp } - if key == metadataFieldKey { + if key == MetadataFieldKey { return ErrAlterMetadataKey } var deleted bool has, _ := e.original.HasKey(key) - if has { - if key == e.rootKey(key) { + rootKey := e.rootKey(key) + if key == rootKey { // if we're trying to delete a root-level key // it must be deleted from the `original` event when `Apply` is called // and from the `pending` event (below) if a value with the same key was put there. - e.markAsDeleted(key) + e.markAsDeleted(rootKey) deleted = true } else { // if it's not a root-level key, we checkout this entire root-level map - // and delete from the `pending` event instead. - e.checkout(key, checkoutModeOnlyMaps) + // and delete from the copy in `pending` event instead. + e.checkout(rootKey, checkoutModeOnlyMaps) } } @@ -212,26 +315,59 @@ func (e *EventEditor) Delete(key string) error { } } -// DeepUpdate implements the `EventAccessor` interface. +// DeleteAll marks all data to be deleted from the event when `Apply` is called. +func (e *EventEditor) DeleteAll() { + // reset all the changes because + // they don't make sense anymore + e.Reset() + // now we mark all root level keys for deletion + // in both `Meta` and `Fields` maps of the original event + if e.original.Meta != nil { + for key := range e.original.Meta { + e.markAsDeleted(metadataKeyPrefix + key) + } + } + if e.original.Fields != nil { + for key := range e.original.Fields { + e.markAsDeleted(key) + } + } +} + +// DeepUpdate recursively copies the key-value pairs from `d` to various properties of the event. +// When the key equals `@timestamp` it's set as the `Timestamp` property of the event. +// When the key equals `@metadata` the update is routed into the `Meta` map instead of `Fields` +// The rest of the keys are set to the `Fields` map. +// If the key is present and the value is a map as well, the sub-map will be updated recursively +// via `DeepUpdate`. +// `DeepUpdateNoOverwrite` is a version of this function that does not +// overwrite existing values. func (e *EventEditor) DeepUpdate(d mapstr.M) { e.deepUpdate(d, updateModeOverwrite) } -// DeepUpdateNoOverwrite implements the `EventAccessor` interface. +// DeepUpdateNoOverwrite recursively copies the key-value pairs from `d` to various properties of the event. +// The `@timestamp` update is ignored due to "no overwrite" behavior. +// When the key equals `@metadata` the update is routed into the `Meta` map instead of `Fields`. +// The rest of the keys are set to the `Fields` map. +// If the key is present and the value is a map as well, the sub-map will be updated recursively +// via `DeepUpdateNoOverwrite`. +// `DeepUpdate` is a version of this function that overwrites existing values. func (e *EventEditor) DeepUpdateNoOverwrite(d mapstr.M) { e.deepUpdate(d, updateModeNoOverwrite) } // Apply write all the changes to the original event making sure that none of the original -// nested maps are modified but replaced with modified clones. +// nested maps are modified but replaced with modified copies. func (e *EventEditor) Apply() { - if e.pending == nil { - return - } - defer e.Reset() - e.original.Timestamp = e.pending.Timestamp for deletedKey := range e.deletions { _ = e.original.Delete(deletedKey) } + if e.pending == nil { + return + } + + e.original.Timestamp = e.pending.Timestamp + // it's enough to overwrite the root-level because // of the checkout mechanism used earlier if len(e.pending.Meta) > 0 { @@ -270,6 +406,107 @@ func (e *EventEditor) Reset() { } } +// AddTags appends a tag to the tags field of the event. If the tags field does not +// exist then it will be created. If the tags field exists and is not a []string +// then an error will be returned. It does not deduplicate the list of tags. +func (e *EventEditor) AddTags(tags ...string) error { + return e.AddTagsWithKey(mapstr.TagsKey, tags...) +} + +// AddTagsWithKey appends a tag to the given field of the event. If the tags field does not +// exist then it will be created. If the tags field exists and is not a []string +// then an error will be returned. It does not deduplicate the list of tags. +func (e *EventEditor) AddTagsWithKey(key string, tags ...string) error { + if len(tags) == 0 { + return nil + } + e.checkout(key, checkoutModeIncludeValues) + e.allocatePending() + if e.pending.Fields == nil { + e.pending.Fields = mapstr.M{} + } + return mapstr.AddTagsWithKey(e.pending.Fields, key, tags) +} + +// AddError appends an error to the event. If the error field does not +// exist then it will be created. +// If the error field exists and another error was already set, it will be converted to a list +// of errors and new errors will be appended there. +// If there is only one error to add, it will be added directly without a list. +func (e *EventEditor) AddError(ee ...EventError) { + if len(ee) == 0 { + return + } + e.checkout(ErrorFieldKey, checkoutModeIncludeValues) + e.allocatePending() + if e.pending.Fields == nil { + e.pending.Fields = mapstr.M{} + } + + var list []mapstr.M + val, err := e.pending.Fields.GetValue(ErrorFieldKey) + // if the value does not exist yet, we initialize the list of errors + // with the size of the current arguments. + if errors.Is(err, mapstr.ErrKeyNotFound) { + list = make([]mapstr.M, 0, len(ee)) + } else if err != nil { + return + } + // if the value already exists, depending on its type + // we need to reformat it or just append to an existing list. + switch typed := val.(type) { + // there was a single error map, should be replaced with a list, + // the existing error will become an item on this list + case mapstr.M: + list = make([]mapstr.M, 0, len(ee)+1) + list = append(list, typed) + // same as the previous case but typed differently + case map[string]interface{}: + list = make([]mapstr.M, 0, len(ee)+1) + list = append(list, mapstr.M(typed)) + // some code can assign an error as a string + // it's not really expected but we should not lose this value + // so, we convert it into a proper error format. + case string: + list = make([]mapstr.M, 0, len(ee)+1) + list = append(list, EventError{Message: typed}.toMap()) + } + + // after the list is prepared and contains already existing errors + // we can start copying the arguments, converting them to maps + for _, err := range ee { + // an error without a message is not a valid error + if err.Message == "" { + continue + } + m := err.toMap() + list = append(list, m) + } + + // if after all manipulations we still have a single error + // we convert it back into a map instead of adding a list + if len(list) == 1 { + e.pending.Fields[ErrorFieldKey] = list[0] + } else { + e.pending.Fields[ErrorFieldKey] = list + } +} + +// String returns the string representation of the current editor's state. +// This function is slow and should be used for debugging purposes only. +func (e *EventEditor) String() string { + deleteList := make([]string, 0, len(e.deletions)) + for key := range e.deletions { + deleteList = append(deleteList, key) + } + m := mapstr.M{ + "original": e.original.String(), + "pending": e.pending.String(), + "deletions": deleteList, + } + return m.String() +} + // deepUpdate checks out all the necessary root-level keys before running the deep update. func (e *EventEditor) deepUpdate(d mapstr.M, mode updateMode) { if len(d) == 0 { @@ -282,14 +519,14 @@ func (e *EventEditor) deepUpdate(d mapstr.M, mode updateMode) { // checkout necessary keys from the original event for key := range d { - if key == timestampFieldKey { + if key == TimestampFieldKey { continue } // we never checkout the whole metadata, only root-level keys // which are about to get updated - if key == metadataFieldKey { - metaUpdate := d[metadataFieldKey] + if key == MetadataFieldKey { + metaUpdate := d[MetadataFieldKey] switch m := metaUpdate.(type) { case mapstr.M: for innerKey := range m { @@ -313,85 +550,140 @@ func (e *EventEditor) deepUpdate(d mapstr.M, mode updateMode) { // markAsDeleted marks a key for deletion when `Apply` is called // if it's a root-level key in the original event. func (e *EventEditor) markAsDeleted(key string) { - dotIdx := e.dotIdx(key) - // nested keys are not marked since nested maps - // are cloned into the `pending` event and altered there - if dotIdx != -1 { + if key == TimestampFieldKey { return } + rootKey := e.rootKey(key) if e.deletions == nil { e.deletions = make(map[string]struct{}) } - e.deletions[key] = struct{}{} + e.deletions[rootKey] = struct{}{} } // checkout clones a nested map of the original event to the event with pending changes. // // If the key leads to a value nested in a map, we checkout the root-level nested map which means -// the whole sub-tree is recursively cloned, so it can be safely modified. +// the whole sub-tree is recursively cloned, so it can be safely modified later. func (e *EventEditor) checkout(key string, mode checkoutMode) { + if key == TimestampFieldKey { + return + } // we're always looking only at the root-level rootKey := e.rootKey(key) + // if the key is not in the original map, the value is empty + if rootKey == "" { + return + } - if e.pending != nil { - // it might be already checked out - checkedOut, _ := e.pending.HasKey(rootKey) - if checkedOut { + e.allocatePending() + + var dstMap, srcMap mapstr.M + if strings.HasPrefix(rootKey, metadataKeyPrefix) { + if e.original.Meta == nil { + return + } + if e.pending.Meta == nil { + e.pending.Meta = mapstr.M{} + } + dstMap = e.pending.Meta + srcMap = e.original.Meta + rootKey = rootKey[metadataKeyOffset:] + } else { + if e.original.Fields == nil { return } + if e.pending.Fields == nil { + e.pending.Fields = mapstr.M{} + } + dstMap = e.pending.Fields + srcMap = e.original.Fields } - // if there is nothing to checkout - return - value, err := e.original.GetValue(rootKey) - if err != nil { + // it might be already checked out + _, checkedOut := dstMap[rootKey] + if checkedOut { return } - e.allocatePending() + // if there is nothing to checkout - return + value, exists := srcMap[rootKey] + if !exists { + return + } // we check out only nested maps, and leave root-level value types in the original map // unless the special `includeValues` mode engaged (used for DeepUpdateNoOverwrite). switch typedVal := value.(type) { case mapstr.M: - _, _ = e.pending.PutValue(rootKey, typedVal.Clone()) + dstMap[rootKey] = typedVal.Clone() case map[string]interface{}: - _, _ = e.pending.PutValue(rootKey, mapstr.M(typedVal).Clone()) + dstMap[rootKey] = mapstr.M(typedVal).Clone() + // TODO slices? default: if mode == checkoutModeIncludeValues { - _, _ = e.pending.PutValue(rootKey, typedVal) + dstMap[rootKey] = typedVal } } } -// dotIdx returns index of the first `.` character or `-1` if there is no `.` character. +// rootKey reduces the key of the original event to its root-level. +// Keys may have `.` character in them on every level and `.` can also mean a nested depth. +// // Accounts for the `@metadata` subkeys, since it's stored in a separate map, // root-level keys will be in the `@metadata.*` namespace. -func (e *EventEditor) dotIdx(key string) int { - // metadata keys are special, since they're stored in a separate map - // we don't want to copy the whole map with all metadata, we want - // to checkout only nested maps one by one for efficiency +// +// Returns empty string if the key does not exist in the original event. +func (e *EventEditor) rootKey(key string) string { + if key == TimestampFieldKey { + return key + } + var ( + prefix string + m mapstr.M + ) + if strings.HasPrefix(key, metadataKeyPrefix) { - // we start looking after the `@metadata` prefix - dotIdx := strings.Index(key[metadataKeyOffset:], ".") - // if there is no dot in the subkey, then the second segment - // is considered to be a root-level key - if dotIdx == -1 { - return -1 - } - // otherwise we need to offset the dot index by the prefix we removed - return dotIdx + metadataKeyOffset + prefix = metadataKeyPrefix + key = key[metadataKeyOffset:] + m = e.original.Meta + } else { + m = e.original.Fields } - return strings.Index(key, ".") -} + // there is no root-level key with this name + if m == nil { + return "" + } -// rootKey reduces the key to its root-level. -func (e *EventEditor) rootKey(key string) string { - dotIdx := e.dotIdx(key) + // this key may be simple and does not contain dots + dotIdx := strings.IndexRune(key, '.') if dotIdx == -1 { - return key - } else { - return key[:dotIdx] + return prefix + key + } + + // fast lane for the whole key + _, exists := m[key] + if exists { + return prefix + key + } + + // otherwise we start with the first segment + // and keep adding the next segment to the key until the resulting value + // exists on the root level. + var rootKey string + for { + rootKey = key[:dotIdx] + _, exists = m[rootKey] + if exists { + return prefix + rootKey + } + dotIdx = strings.IndexRune(key[dotIdx+1:], '.') + // we checked above for the full key and it didn't exist, + // no need to check again, we return since the key is not found + if dotIdx == -1 { + return "" + } + dotIdx += len(rootKey) + 1 } } @@ -404,7 +696,7 @@ func (e *EventEditor) checkDeleted(key string) bool { return deleted } -// allocatePending makes sure that the `pending` event is allocated for collecting changes. +// allocatePending makes sure that the `pending` event is allocated for collecting new changes. func (e *EventEditor) allocatePending() { if e.pending != nil { return diff --git a/libbeat/beat/event_editor_test.go b/libbeat/beat/event_editor_test.go index 05b670f675a9..6af12ab4bff5 100644 --- a/libbeat/beat/event_editor_test.go +++ b/libbeat/beat/event_editor_test.go @@ -49,6 +49,7 @@ func TestEventEditor(t *testing.T) { event := &Event{ Timestamp: time.Now(), Meta: mapstr.M{ + "a.b": "c", "metaLevel0Map": metadataNestedMap, "metaLevel0Value": "metavalue1", // this key should never be edited by the tests @@ -58,6 +59,7 @@ func TestEventEditor(t *testing.T) { "metaUntouchedMap": metaUntouchedMap, }, Fields: mapstr.M{ + "a.b": "c", "fieldsLevel0Map": fieldsNestedMap, "fieldsLevel0Value": "fieldsvalue1", // this key should never be edited by the tests @@ -69,25 +71,56 @@ func TestEventEditor(t *testing.T) { } t.Run("rootKey", func(t *testing.T) { + event := event.Clone() + event.Meta["some.dot.metakey"] = mapstr.M{ + "that.should": mapstr.M{ + "be": "supported", + }, + } + event.Fields["some.dot.key"] = mapstr.M{ + "that.should": mapstr.M{ + "be": "supported", + }, + } cases := []struct { val string exp string }{ { - val: "@metadata.some", - exp: "@metadata.some", + val: "@metadata.a.b", + exp: "@metadata.a.b", + }, + { + val: "@metadata.metaLevel0Value", + exp: "@metadata.metaLevel0Value", + }, + { + val: "@metadata.metaLevel0Map.metaLevel1Map", + exp: "@metadata.metaLevel0Map", + }, + { + val: "@metadata.some.dot.metakey.that.should.be", + exp: "@metadata.some.dot.metakey", }, { - val: "@metadata.some.nested", - exp: "@metadata.some", + val: "a.b", + exp: "a.b", }, { - val: "some.nested.key", - exp: "some", + val: "fieldsLevel0Map.fieldsLevel1Value", + exp: "fieldsLevel0Map", }, { - val: "key", - exp: "key", + val: "fieldsLevel0Map.fieldsLevel1Map.fieldsLevel2Value", + exp: "fieldsLevel0Map", + }, + { + val: "fieldsLevel0Value", + exp: "fieldsLevel0Value", + }, + { + val: "some.dot.key.that.should.be", + exp: "some.dot.key", }, } @@ -196,6 +229,13 @@ func TestEventEditor(t *testing.T) { }) }) + t.Run("gets a root-level dot-key", func(t *testing.T) { + editor := NewEventEditor(event) + requireMapValues(t, editor, map[string]interface{}{ + "@metadata.a.b": "c", + }) + }) + t.Run("gets a nested map", func(t *testing.T) { editor := NewEventEditor(event) nested, err := editor.GetValue("@metadata.metaLevel0Map") @@ -249,6 +289,13 @@ func TestEventEditor(t *testing.T) { }) }) + t.Run("gets a root-level dot-key", func(t *testing.T) { + editor := NewEventEditor(event) + requireMapValues(t, editor, map[string]interface{}{ + "a.b": "c", + }) + }) + t.Run("gets a nested map", func(t *testing.T) { editor := NewEventEditor(event) nested, err := editor.GetValue("fieldsLevel0Map") @@ -339,6 +386,29 @@ func TestEventEditor(t *testing.T) { require.Equal(t, value, val3) }) + t.Run("root-level dot-key", func(t *testing.T) { + editor := NewEventEditor(event) + key := "@metadata.a.b" + value := "c" + + val1, err := editor.GetValue(key) + require.NoError(t, err) + require.Equal(t, value, val1) + + err = editor.Delete(key) + require.NoError(t, err) + + val2, err := editor.GetValue(key) + require.Error(t, err) + require.ErrorIs(t, err, mapstr.ErrKeyNotFound) + require.Nil(t, val2) + + // checking if the original event still has it + val3, err := event.GetValue(key) + require.NoError(t, err) + require.Equal(t, value, val3) + }) + t.Run("root-level map", func(t *testing.T) { editor := NewEventEditor(event) key := "@metadata.metaLevel0Map" @@ -452,6 +522,29 @@ func TestEventEditor(t *testing.T) { require.Equal(t, value, val3) }) + t.Run("root-level dot-key", func(t *testing.T) { + editor := NewEventEditor(event) + key := "a.b" + value := "c" + + val1, err := editor.GetValue(key) + require.NoError(t, err) + require.Equal(t, value, val1) + + err = editor.Delete(key) + require.NoError(t, err) + + val2, err := editor.GetValue(key) + require.Error(t, err) + require.ErrorIs(t, err, mapstr.ErrKeyNotFound) + require.Nil(t, val2) + + // checking if the original event still has it + val3, err := event.GetValue(key) + require.NoError(t, err) + require.Equal(t, value, val3) + }) + t.Run("root-level map", func(t *testing.T) { editor := NewEventEditor(event) key := "fieldsLevel0Map" @@ -595,7 +688,27 @@ func TestEventEditor(t *testing.T) { require.ErrorIs(t, err, mapstr.ErrKeyNotFound) }) - t.Run("new nested value", func(t *testing.T) { + t.Run("existing root-level dot-key", func(t *testing.T) { + editor := NewEventEditor(event) + key := "@metadata.a.b" + value := mapstr.M{ + "some": "value", + } + prevVal, err := editor.PutValue(key, value) + require.NoError(t, err) + require.Equal(t, "c", prevVal) + + val, err := editor.GetValue(key) + require.NoError(t, err) + require.Equal(t, value, val) + + // the original event should have the previous value + val, err = event.GetValue(key) + require.NoError(t, err) + require.Equal(t, "c", val) + }) + + t.Run("new nested value in existing map", func(t *testing.T) { editor := NewEventEditor(event) key := "@metadata.metaLevel0Map.metaLevel1Map.new" value := "newvalue" @@ -619,6 +732,25 @@ func TestEventEditor(t *testing.T) { requireNotSameMap(t, metadataNestedMap, val) }) + t.Run("absolutely new nested value", func(t *testing.T) { + editor := NewEventEditor(event) + key := "@metadata.new1.new2.new3" + value := "newvalue" + + prevVal, err := editor.PutValue(key, value) + require.NoError(t, err) + require.Nil(t, prevVal) + + val, err := editor.GetValue(key) + require.NoError(t, err) + require.Equal(t, value, val) + + // the original event should not have it + _, err = event.GetValue(key) + require.Error(t, err) + require.ErrorIs(t, err, mapstr.ErrKeyNotFound) + }) + t.Run("new nested map", func(t *testing.T) { editor := NewEventEditor(event) key := "@metadata.metaLevel0Map.metaLevel1Map.new" @@ -710,7 +842,27 @@ func TestEventEditor(t *testing.T) { require.ErrorIs(t, err, mapstr.ErrKeyNotFound) }) - t.Run("new nested value", func(t *testing.T) { + t.Run("existing root-level dot-key", func(t *testing.T) { + editor := NewEventEditor(event) + key := "a.b" + value := mapstr.M{ + "some": "value", + } + prevVal, err := editor.PutValue(key, value) + require.NoError(t, err) + require.Equal(t, "c", prevVal) + + val, err := editor.GetValue(key) + require.NoError(t, err) + require.Equal(t, value, val) + + // the original event should have the previous value + val, err = event.GetValue(key) + require.NoError(t, err) + require.Equal(t, "c", val) + }) + + t.Run("new nested value in existing map", func(t *testing.T) { editor := NewEventEditor(event) key := "fieldsLevel0Map.fieldsLevel1Map.new" value := "newvalue" @@ -731,7 +883,26 @@ func TestEventEditor(t *testing.T) { // the original `fieldsLevel0Map` should be cloned/checkedout now val, err = editor.GetValue("fieldsLevel0Map") require.NoError(t, err) - requireNotSameMap(t, metadataNestedMap, val) + requireNotSameMap(t, fieldsNestedMap, val) + }) + + t.Run("absolutely new nested value", func(t *testing.T) { + editor := NewEventEditor(event) + key := "new1.new2.new3" + value := "newvalue" + + prevVal, err := editor.PutValue(key, value) + require.NoError(t, err) + require.Nil(t, prevVal) + + val, err := editor.GetValue(key) + require.NoError(t, err) + require.Equal(t, value, val) + + // the original event should not have it + _, err = event.GetValue(key) + require.Error(t, err) + require.ErrorIs(t, err, mapstr.ErrKeyNotFound) }) t.Run("new nested map", func(t *testing.T) { @@ -785,6 +956,27 @@ func TestEventEditor(t *testing.T) { requireNotSameMap(t, fieldsNestedMap, val) }) }) + + t.Run("type conflict", func(t *testing.T) { + event := &Event{ + Meta: mapstr.M{ + "a": 9, + "c": 10, + }, + Fields: mapstr.M{ + "a": 9, + "c": 10, + }, + } + + editor := NewEventEditor(event) + _, err := editor.PutValue("a.c", 10) + require.Error(t, err) + require.Equal(t, "expected map but type is int", err.Error()) + _, err = editor.PutValue("a.value", 9) + require.Error(t, err) + require.Equal(t, "expected map but type is int", err.Error()) + }) }) t.Run("Apply", func(t *testing.T) { @@ -836,6 +1028,7 @@ func TestEventEditor(t *testing.T) { expEvent := &Event{ Timestamp: newTs, Meta: mapstr.M{ + "a.b": "c", "metaLevel0Map": mapstr.M{ "metaLevel1Map": mapstr.M{ "new1": "newmetavalue1", @@ -847,6 +1040,7 @@ func TestEventEditor(t *testing.T) { "metaUntouchedMap": metaUntouchedMap, }, Fields: mapstr.M{ + "a.b": "c", "fieldsLevel0Map": mapstr.M{ "fieldsLevel1Map": mapstr.M{ "new3": "newfieldsvalue1", @@ -983,6 +1177,7 @@ func TestEventEditor(t *testing.T) { expEvent := &Event{ Timestamp: newTs, Meta: mapstr.M{ + "a.b": "c", "metaLevel0Map": mapstr.M{ "metaLevel1Map": mapstr.M{ "metaLevel2Value": "metavalue3", @@ -995,6 +1190,7 @@ func TestEventEditor(t *testing.T) { "metaUntouchedMap": metaUntouchedMap, }, Fields: mapstr.M{ + "a.b": "c", "fieldsLevel0Map": mapstr.M{ "fieldsLevel1Map": mapstr.M{ "fieldsLevel2Value": "fieldsvalue3", @@ -1032,6 +1228,7 @@ func TestEventEditor(t *testing.T) { // should have the original/non-overwritten timestamp value Timestamp: event.Timestamp, Meta: mapstr.M{ + "a.b": "c", "metaLevel0Map": mapstr.M{ "metaLevel1Map": mapstr.M{ "metaLevel2Value": "metavalue3", @@ -1044,6 +1241,7 @@ func TestEventEditor(t *testing.T) { "metaUntouchedMap": metaUntouchedMap, }, Fields: mapstr.M{ + "a.b": "c", "fieldsLevel0Map": mapstr.M{ "fieldsLevel1Map": mapstr.M{ "fieldsLevel2Value": "fieldsvalue3", @@ -1072,6 +1270,32 @@ func TestEventEditor(t *testing.T) { requireClonedMaps(t, expEvent.Fields, cloned.Fields) }) }) + + t.Run("hierarchy", func(t *testing.T) { + event := &Event{ + Fields: mapstr.M{ + "a.b": 1, + }, + } + editor := NewEventEditor(event) + err := editor.Delete("a.b") + require.NoError(t, err) + + prev, err := editor.PutValue("a.b.c", 1) + require.NoError(t, err) + require.Nil(t, prev) + + expFields := mapstr.M{ + "a": mapstr.M{ + "b": mapstr.M{ + "c": 1, + }, + }, + } + + editor.Apply() + requireClonedMaps(t, expFields, event.Fields) + }) } func requireClonedMaps(t *testing.T, expected, actual interface{}) { @@ -1100,7 +1324,7 @@ func requireNotSameMap(t *testing.T, expected, actual interface{}) { require.NotEqualf(t, expectedAddr, actualAddr, "should reference different maps: %s == %s", expectedAddr, actualAddr) } -func requireMapValues(t *testing.T, e EventAccessor, expected map[string]interface{}) { +func requireMapValues(t *testing.T, e *EventEditor, expected map[string]interface{}) { t.Helper() for key := range expected { val, err := e.GetValue(key) @@ -1108,3 +1332,13 @@ func requireMapValues(t *testing.T, e EventAccessor, expected map[string]interfa require.Equal(t, expected[key], val) } } + +func (e *Event) Clone() *Event { + return &Event{ + Timestamp: e.Timestamp, + Meta: e.Meta.Clone(), + Fields: e.Fields.Clone(), + Private: e.Private, + TimeSeries: e.TimeSeries, + } +} diff --git a/libbeat/beat/event_test.go b/libbeat/beat/event_test.go index 2719567c1acb..0c09eb7345e9 100644 --- a/libbeat/beat/event_test.go +++ b/libbeat/beat/event_test.go @@ -117,7 +117,7 @@ func TestDeepUpdate(t *testing.T) { name: "updates timestamp", event: newEvent(mapstr.M{}), update: mapstr.M{ - timestampFieldKey: ts, + TimestampFieldKey: ts, }, mode: updateModeOverwrite, expected: &Event{ @@ -133,7 +133,7 @@ func TestDeepUpdate(t *testing.T) { "Timestamp": ts, }), update: mapstr.M{ - timestampFieldKey: time.Now().Add(time.Hour), + TimestampFieldKey: time.Now().Add(time.Hour), }, mode: updateModeNoOverwrite, expected: &Event{ @@ -147,7 +147,7 @@ func TestDeepUpdate(t *testing.T) { name: "initializes metadata if nil", event: newEvent(mapstr.M{}), update: mapstr.M{ - metadataFieldKey: mapstr.M{ + MetadataFieldKey: mapstr.M{ "first": "new", "second": 42, }, @@ -170,7 +170,7 @@ func TestDeepUpdate(t *testing.T) { }, }), update: mapstr.M{ - metadataFieldKey: mapstr.M{ + MetadataFieldKey: mapstr.M{ "first": "new", "second": 42, }, @@ -194,7 +194,7 @@ func TestDeepUpdate(t *testing.T) { }, }), update: mapstr.M{ - metadataFieldKey: mapstr.M{ + MetadataFieldKey: mapstr.M{ "first": "new", "second": 42, }, diff --git a/libbeat/beat/pipeline.go b/libbeat/beat/pipeline.go index 8e8b285042c4..bb409a7a9d9f 100644 --- a/libbeat/beat/pipeline.go +++ b/libbeat/beat/pipeline.go @@ -155,7 +155,7 @@ type ProcessorList interface { // registered with the publisher pipeline. type Processor interface { String() string // print full processor description - Run(in *Event) (event *Event, err error) + Run(*EventEditor) (dropped bool, err error) } // PublishMode enum sets some requirements on the client connection to the beats diff --git a/libbeat/common/jsontransform/expand.go b/libbeat/common/jsontransform/expand.go index be07c4200740..3cd6f4db8565 100644 --- a/libbeat/common/jsontransform/expand.go +++ b/libbeat/common/jsontransform/expand.go @@ -31,10 +31,12 @@ import ( // conflicts (i.e. a common prefix where one field is an object and another // is a non-object), an error key is added to the event if add_error_key // is enabled. -func ExpandFields(logger *logp.Logger, event *beat.Event, m mapstr.M, addErrorKey bool) { +func ExpandFields(logger *logp.Logger, event *beat.EventEditor, m mapstr.M, addErrorKey bool) { if err := expandFields(m); err != nil { logger.Errorf("JSON: failed to expand fields: %s", err) - event.SetErrorWithOption(err.Error(), addErrorKey, "", "") + if addErrorKey { + event.AddError(beat.EventError{Message: err.Error()}) + } } } diff --git a/libbeat/common/jsontransform/jsonhelper.go b/libbeat/common/jsontransform/jsonhelper.go index 15d70ae929ac..b08625b709d0 100644 --- a/libbeat/common/jsontransform/jsonhelper.go +++ b/libbeat/common/jsontransform/jsonhelper.go @@ -23,7 +23,6 @@ import ( "time" "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/elastic-agent-libs/mapstr" ) const ( @@ -37,78 +36,68 @@ var ( ) // WriteJSONKeys writes the json keys to the given event based on the overwriteKeys option and the addErrKey -func WriteJSONKeys(event *beat.Event, keys map[string]interface{}, expandKeys, overwriteKeys, addErrKey bool) { +func WriteJSONKeys(event *beat.EventEditor, keys map[string]interface{}, expandKeys, overwriteKeys, addErrKey bool) { + setError := func(string, string) {} + if addErrKey { + setError = func(msg, field string) { + event.AddError(beat.EventError{Message: msg, Field: field}) + } + } if expandKeys { if err := expandFields(keys); err != nil { - event.SetErrorWithOption(err.Error(), addErrKey, "", "") + setError(err.Error(), "") return } } if !overwriteKeys { - // @timestamp and @metadata fields are root-level fields. We remove them so they - // don't become part of event.Fields. - removeKeys(keys, "@timestamp", "@metadata") - // Then, perform deep update without overwriting - event.Fields.DeepUpdateNoOverwrite(keys) + event.DeepUpdateNoOverwrite(keys) return } for k, v := range keys { switch k { - case "@timestamp": + case beat.TimestampFieldKey: vstr, ok := v.(string) if !ok { - event.SetErrorWithOption("@timestamp not overwritten (not string)", addErrKey, "", "") + setError("not overwritten (not a string)", beat.TimestampFieldKey) + removeKeys(keys, k) continue } // @timestamp must be of format RFC3339 or ISO8601 ts, err := parseTimestamp(vstr) if err != nil { - event.SetErrorWithOption(fmt.Sprintf("@timestamp not overwritten (parse error on %s)", vstr), addErrKey, "", "") + setError(fmt.Sprintf("timestamp parse error on %s", vstr), beat.TimestampFieldKey) + removeKeys(keys, k) continue } - event.Timestamp = ts - - case "@metadata": - switch m := v.(type) { - case map[string]string: - if event.Meta == nil && len(m) > 0 { - event.Meta = mapstr.M{} - } - for meta, value := range m { - event.Meta[meta] = value - } - - case map[string]interface{}: - if event.Meta == nil { - event.Meta = mapstr.M{} - } - event.Meta.DeepUpdate(mapstr.M(m)) - + keys[k] = ts + case beat.MetadataFieldKey: + switch v.(type) { + case map[string]string, map[string]interface{}: default: - event.SetErrorWithOption("failed to update @metadata", addErrKey, "", "") + setError("can't replace with a value", beat.MetadataFieldKey) + removeKeys(keys, k) } - case "type": + case beat.TypeFieldKey: vstr, ok := v.(string) if !ok { - event.SetErrorWithOption("type not overwritten (not string)", addErrKey, "", "") + setError("not overwritten (not a string)", beat.TypeFieldKey) + removeKeys(keys, k) continue } if len(vstr) == 0 || vstr[0] == '_' { - event.SetErrorWithOption(fmt.Sprintf("type not overwritten (invalid value [%s])", vstr), addErrKey, "", "") + setError(fmt.Sprintf("not overwritten (invalid value [%s])", vstr), beat.TypeFieldKey) + removeKeys(keys, k) continue } - event.Fields[k] = vstr + keys[k] = vstr } } - // We have accounted for @timestamp, @metadata, type above. So let's remove these keys and - // deep update the event with the rest of the keys. - removeKeys(keys, "@timestamp", "@metadata", "type") - event.Fields.DeepUpdate(keys) + event.DeepUpdate(keys) } func removeKeys(keys map[string]interface{}, names ...string) { diff --git a/libbeat/processors/actions/add_fields.go b/libbeat/processors/actions/add_fields.go index cd37910cbe5e..45f4fa60e273 100644 --- a/libbeat/processors/actions/add_fields.go +++ b/libbeat/processors/actions/add_fields.go @@ -31,7 +31,6 @@ import ( type addFields struct { fields mapstr.M - shared bool overwrite bool } @@ -61,34 +60,26 @@ func CreateAddFields(c *conf.C) (beat.Processor, error) { return makeFieldsProcessor( optTarget(config.Target, FieldsKey), config.Fields, - true, ), nil } // NewAddFields creates a new processor adding the given fields to events. -// Set `shared` true if there is the chance of labels being changed/modified by -// subsequent processors. -func NewAddFields(fields mapstr.M, shared bool, overwrite bool) beat.Processor { - return &addFields{fields: fields, shared: shared, overwrite: overwrite} +func NewAddFields(fields mapstr.M, overwrite bool) beat.Processor { + return &addFields{fields: fields, overwrite: overwrite} } -func (af *addFields) Run(event *beat.Event) (*beat.Event, error) { +func (af *addFields) Run(event *beat.EventEditor) (dropped bool, err error) { if event == nil || len(af.fields) == 0 { - return event, nil - } - - fields := af.fields - if af.shared { - fields = fields.Clone() + return false, nil } if af.overwrite { - event.DeepUpdate(fields) + event.DeepUpdate(af.fields) } else { - event.DeepUpdateNoOverwrite(fields) + event.DeepUpdateNoOverwrite(af.fields) } - return event, nil + return false, nil } func (af *addFields) String() string { @@ -103,12 +94,12 @@ func optTarget(opt *string, def string) string { return *opt } -func makeFieldsProcessor(target string, fields mapstr.M, shared bool) beat.Processor { +func makeFieldsProcessor(target string, fields mapstr.M) beat.Processor { if target != "" { fields = mapstr.M{ target: fields, } } - return NewAddFields(fields, shared, true) + return NewAddFields(fields, true) } diff --git a/libbeat/processors/actions/add_labels.go b/libbeat/processors/actions/add_labels.go index c282cf470b4d..df6c005c8ed0 100644 --- a/libbeat/processors/actions/add_labels.go +++ b/libbeat/processors/actions/add_labels.go @@ -51,7 +51,7 @@ func createAddLabels(c *conf.C) (beat.Processor, error) { return nil, fmt.Errorf("failed to flatten labels: %w", err) } - return makeFieldsProcessor(LabelsKey, flatLabels, true), nil + return makeFieldsProcessor(LabelsKey, flatLabels), nil } // NewAddLabels creates a new processor adding the given object to events. Set @@ -60,7 +60,7 @@ func createAddLabels(c *conf.C) (beat.Processor, error) { // If labels contains nested objects, NewAddLabels will flatten keys into labels by // by joining names with a dot ('.') . // The labels will be inserted into the 'labels' field. -func NewAddLabels(labels mapstr.M, shared bool) (beat.Processor, error) { +func NewAddLabels(labels mapstr.M) (beat.Processor, error) { flatLabels, err := flattenLabels(labels) if err != nil { return nil, fmt.Errorf("failed to flatten labels: %w", err) @@ -68,7 +68,7 @@ func NewAddLabels(labels mapstr.M, shared bool) (beat.Processor, error) { return NewAddFields(mapstr.M{ LabelsKey: flatLabels, - }, shared, true), nil + }, true), nil } func flattenLabels(labels mapstr.M) (mapstr.M, error) { diff --git a/libbeat/processors/actions/add_network_direction.go b/libbeat/processors/actions/add_network_direction.go index 295f74c1bef7..3681adeb4b72 100644 --- a/libbeat/processors/actions/add_network_direction.go +++ b/libbeat/processors/actions/add_network_direction.go @@ -61,45 +61,45 @@ func NewAddNetworkDirection(cfg *conf.C) (beat.Processor, error) { return networkDirection, nil } -func (m *networkDirectionProcessor) Run(event *beat.Event) (*beat.Event, error) { +func (m *networkDirectionProcessor) Run(event *beat.EventEditor) (dropped bool, err error) { sourceI, err := event.GetValue(m.Source) if err != nil { //nolint:nilerr // doesn't have the required field value to analyze - return event, nil + return false, nil } source, _ := sourceI.(string) if source == "" { // wrong type or not set - return event, nil + return false, nil } destinationI, err := event.GetValue(m.Destination) if err != nil { //nolint:nilerr // doesn't have the required field value to analyze - return event, nil + return false, nil } destination, _ := destinationI.(string) if destination == "" { // wrong type or not set - return event, nil + return false, nil } sourceIP := net.ParseIP(source) destinationIP := net.ParseIP(destination) if sourceIP == nil || destinationIP == nil { // bad ip address - return event, nil + return false, nil } internalSource, err := conditions.NetworkContains(sourceIP, m.InternalNetworks...) if err != nil { - return event, err + return false, err } internalDestination, err := conditions.NetworkContains(destinationIP, m.InternalNetworks...) if err != nil { - return event, err + return false, err } _, _ = event.PutValue(m.Target, networkDirection(internalSource, internalDestination)) - return event, nil + return false, nil } func networkDirection(internalSource, internalDestination bool) string { diff --git a/libbeat/processors/actions/add_network_direction_test.go b/libbeat/processors/actions/add_network_direction_test.go index 52ee3ba70386..060305cd4565 100644 --- a/libbeat/processors/actions/add_network_direction_test.go +++ b/libbeat/processors/actions/add_network_direction_test.go @@ -63,13 +63,15 @@ func TestNetworkDirection(t *testing.T) { "internal_networks": tt.InternalNetworks, })) require.NoError(t, err) - observed, err := p.Run(&evt) + ed := beat.NewEventEditor(&evt) + dropped, err := p.Run(ed) if tt.Error { require.Error(t, err) } else { require.NoError(t, err) + require.False(t, dropped) } - enriched, err := observed.Fields.GetValue("direction") + enriched, err := ed.GetValue("direction") if tt.Direction == "" { require.Error(t, err) } else { @@ -98,10 +100,14 @@ func TestNetworkDirection(t *testing.T) { expectedMeta := mapstr.M{ "direction": "external", } + expectedFields := evt.Fields.Clone() - observed, err := p.Run(&evt) + ed := beat.NewEventEditor(&evt) + dropped, err := p.Run(ed) require.NoError(t, err) - require.Equal(t, expectedMeta, observed.Meta) - require.Equal(t, evt.Fields, observed.Fields) + require.False(t, dropped) + ed.Apply() + require.Equal(t, expectedMeta, evt.Meta) + require.Equal(t, expectedFields, evt.Fields) }) } diff --git a/libbeat/processors/actions/add_tags.go b/libbeat/processors/actions/add_tags.go index 1cd77b7dd33f..f80ba82db765 100644 --- a/libbeat/processors/actions/add_tags.go +++ b/libbeat/processors/actions/add_tags.go @@ -73,9 +73,9 @@ func NewAddTags(target string, tags []string) beat.Processor { return &addTags{tags: tags, target: target} } -func (at *addTags) Run(event *beat.Event) (*beat.Event, error) { - _ = mapstr.AddTagsWithKey(event.Fields, at.target, at.tags) - return event, nil +func (at *addTags) Run(event *beat.EventEditor) (dropped bool, err error) { + err = event.AddTagsWithKey(at.target, at.tags...) + return false, err } func (at *addTags) String() string { diff --git a/libbeat/processors/actions/append.go b/libbeat/processors/actions/append.go index e7429449db40..d36a9b6eb8b2 100644 --- a/libbeat/processors/actions/append.go +++ b/libbeat/processors/actions/append.go @@ -73,31 +73,22 @@ func NewAppendProcessor(c *conf.C) (beat.Processor, error) { return f, nil } -func (f *appendProcessor) Run(event *beat.Event) (*beat.Event, error) { - var backup *beat.Event - if f.config.FailOnError { - backup = event.Clone() - } - - err := f.appendValues(f.config.TargetField, f.config.Fields, f.config.Values, event) +func (f *appendProcessor) Run(event *beat.EventEditor) (dropped bool, err error) { + err = f.appendValues(f.config.TargetField, f.config.Fields, f.config.Values, event) if err != nil { errMsg := fmt.Errorf("failed to append fields in append processor: %w", err) if publisher.LogWithTrace() { f.logger.Debug(errMsg.Error()) } if f.config.FailOnError { - event = backup - if _, err := event.PutValue("error.message", errMsg.Error()); err != nil { - return nil, fmt.Errorf("failed to append fields in append processor: %w", err) - } - return event, err + return false, beat.EventError{Message: errMsg.Error(), Processor: f.String()} } } - return event, nil + return false, nil } -func (f *appendProcessor) appendValues(target string, fields []string, values []interface{}, event *beat.Event) error { +func (f *appendProcessor) appendValues(target string, fields []string, values []interface{}, event *beat.EventEditor) error { var arr []interface{} // get the existing value of target field diff --git a/libbeat/processors/actions/append_test.go b/libbeat/processors/actions/append_test.go index 8cb8549b389b..2e4afca0fd5a 100644 --- a/libbeat/processors/actions/append_test.go +++ b/libbeat/processors/actions/append_test.go @@ -24,6 +24,7 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" + "github.com/stretchr/testify/assert" ) var log = logp.NewLogger("append_test") @@ -174,7 +175,7 @@ func Test_appendProcessor_appendValues(t *testing.T) { config: tt.fields.config, logger: tt.fields.logger, } - if err := f.appendValues(tt.args.target, tt.args.fields, tt.args.values, tt.args.event); (err != nil) != tt.wantErr { + if err := f.appendValues(tt.args.target, tt.args.fields, tt.args.values, beat.NewEventEditor(tt.args.event)); (err != nil) != tt.wantErr { t.Errorf("appendProcessor.appendValues() error = %v, wantErr %v", err, tt.wantErr) } }) @@ -195,6 +196,7 @@ func Test_appendProcessor_Run(t *testing.T) { args args want *beat.Event wantErr bool + wantErrMsg string }{ { description: "positive flow", @@ -378,14 +380,11 @@ func Test_appendProcessor_Run(t *testing.T) { Fields: mapstr.M{}, }, }, - wantErr: true, + wantErr: true, + wantErrMsg: "[processor=append=target] failed to append fields in append processor: could not fetch value for key: missing-field, Error: key not found", want: &beat.Event{ - Meta: mapstr.M{}, - Fields: mapstr.M{ - "error": mapstr.M{ - "message": "failed to append fields in append processor: could not fetch value for key: missing-field, Error: key not found", - }, - }, + Meta: mapstr.M{}, + Fields: mapstr.M{}, }, }, } @@ -395,14 +394,17 @@ func Test_appendProcessor_Run(t *testing.T) { config: tt.fields.config, logger: tt.fields.logger, } - got, err := f.Run(tt.args.event) - if (err != nil) != tt.wantErr { - t.Errorf("appendProcessor.Run() error = %v, wantErr %v", err, tt.wantErr) + ed := beat.NewEventEditor(tt.args.event) + _, err := f.Run(ed) + if tt.wantErr { + assert.Error(t, err) + if tt.wantErrMsg != "" { + assert.Equal(t, tt.wantErrMsg, err.Error()) + } return } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("appendProcessor.Run() = %v, want %v", got, tt.want) - } + ed.Apply() + assert.Equal(t, tt.want, tt.args.event) }) } } diff --git a/libbeat/processors/actions/common_test.go b/libbeat/processors/actions/common_test.go index e9d25708e701..f1d98e484eba 100644 --- a/libbeat/processors/actions/common_test.go +++ b/libbeat/processors/actions/common_test.go @@ -60,17 +60,18 @@ func testProcessors(t *testing.T, cases map[string]testCase) { if test.eventMeta != nil { current.Meta = test.eventMeta.Clone() } + ed := beat.NewEventEditor(current) for i, processor := range ps { var err error - current, err = processor.Run(current) + dropped, err := processor.Run(ed) if err != nil { t.Fatal(err) } - if current == nil { + if dropped { t.Fatalf("Event dropped(%v)", i) } } - + ed.Apply() assert.Equal(t, test.wantFields, current.Fields) assert.Equal(t, test.wantMeta, current.Meta) }) diff --git a/libbeat/processors/actions/copy_fields.go b/libbeat/processors/actions/copy_fields.go index 373c939388cb..a57c86ca2b40 100644 --- a/libbeat/processors/actions/copy_fields.go +++ b/libbeat/processors/actions/copy_fields.go @@ -69,12 +69,7 @@ func NewCopyFields(c *conf.C) (beat.Processor, error) { return f, nil } -func (f *copyFields) Run(event *beat.Event) (*beat.Event, error) { - var backup *beat.Event - if f.config.FailOnError { - backup = event.Clone() - } - +func (f *copyFields) Run(event *beat.EventEditor) (dropped bool, err error) { for _, field := range f.config.Fields { err := f.copyField(field.From, field.To, event) if err != nil { @@ -83,17 +78,15 @@ func (f *copyFields) Run(event *beat.Event) (*beat.Event, error) { f.logger.Debug(errMsg.Error()) } if f.config.FailOnError { - event = backup - _, _ = event.PutValue("error.message", errMsg.Error()) - return event, err + return false, beat.EventError{Message: errMsg.Error(), Processor: f.String()} } } } - return event, nil + return false, nil } -func (f *copyFields) copyField(from string, to string, event *beat.Event) error { +func (f *copyFields) copyField(from string, to string, event *beat.EventEditor) error { _, err := event.GetValue(to) if err == nil { return fmt.Errorf("target field %s already exists, drop or rename this field first", to) diff --git a/libbeat/processors/actions/copy_fields_test.go b/libbeat/processors/actions/copy_fields_test.go index 978cd24ee6a1..aa38a86f08f1 100644 --- a/libbeat/processors/actions/copy_fields_test.go +++ b/libbeat/processors/actions/copy_fields_test.go @@ -161,11 +161,12 @@ func TestCopyFields(t *testing.T) { event := &beat.Event{ Fields: test.Input, } - - newEvent, err := p.Run(event) + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) assert.NoError(t, err) - - assert.Equal(t, test.Expected, newEvent.Fields) + assert.False(t, dropped) + ed.Apply() + assert.Equal(t, test.Expected, event.Fields) }) } @@ -187,16 +188,19 @@ func TestCopyFields(t *testing.T) { "message": "please copy this line", }, } + ed := beat.NewEventEditor(event) expMeta := mapstr.M{ "message": "please copy this line", "message_copied": "please copy this line", } - newEvent, err := p.Run(event) + dropped, err := p.Run(ed) assert.NoError(t, err) + assert.False(t, dropped) - assert.Equal(t, expMeta, newEvent.Meta) - assert.Equal(t, event.Fields, newEvent.Fields) + ed.Apply() + assert.Equal(t, expMeta, event.Meta) + assert.Equal(t, event.Fields, event.Fields) }) } diff --git a/libbeat/processors/actions/decode_base64_field.go b/libbeat/processors/actions/decode_base64_field.go index fdfd038b05d6..b17c4d94b7d2 100644 --- a/libbeat/processors/actions/decode_base64_field.go +++ b/libbeat/processors/actions/decode_base64_field.go @@ -74,33 +74,26 @@ func NewDecodeBase64Field(c *cfg.C) (beat.Processor, error) { }, nil } -func (f *decodeBase64Field) Run(event *beat.Event) (*beat.Event, error) { - var backup *beat.Event - // Creates a copy of the event to revert in case of failure - if f.config.FailOnError { - backup = event.Clone() - } - - err := f.decodeField(event) +func (f *decodeBase64Field) Run(event *beat.EventEditor) (dropped bool, err error) { + err = f.decodeField(event) if err != nil { errMsg := fmt.Errorf("failed to decode base64 fields in processor: %w", err) if publisher.LogWithTrace() { f.log.Debug(errMsg.Error()) } if f.config.FailOnError { - event = backup - _, _ = event.PutValue("error.message", errMsg.Error()) - return event, err + event.AddError(beat.EventError{Message: errMsg.Error()}) + return false, err } } - return event, nil + return false, nil } func (f decodeBase64Field) String() string { return fmt.Sprintf("%s=%+v", processorName, f.config.Field) } -func (f *decodeBase64Field) decodeField(event *beat.Event) error { +func (f *decodeBase64Field) decodeField(event *beat.EventEditor) error { value, err := event.GetValue(f.config.Field.From) if err != nil { if f.config.IgnoreMissing && errors.Is(err, mapstr.ErrKeyNotFound) { diff --git a/libbeat/processors/actions/decode_base64_field_test.go b/libbeat/processors/actions/decode_base64_field_test.go index 6e6261446e33..c3c54cb3c494 100644 --- a/libbeat/processors/actions/decode_base64_field_test.go +++ b/libbeat/processors/actions/decode_base64_field_test.go @@ -211,14 +211,16 @@ func TestDecodeBase64Run(t *testing.T) { Fields: test.Input, } - newEvent, err := f.Run(event) + ed := beat.NewEventEditor(event) + dropped, err := f.Run(ed) if !test.error { assert.NoError(t, err) } else { assert.Error(t, err) } - - assert.Equal(t, test.Output, newEvent.Fields) + assert.False(t, dropped) + ed.Apply() + assert.Equal(t, test.Output, event.Fields) }) } @@ -236,6 +238,7 @@ func TestDecodeBase64Run(t *testing.T) { "field1": "Y29ycmVjdCBkYXRh", }, } + ed := beat.NewEventEditor(event) f := &decodeBase64Field{ log: logp.NewLogger(processorName), @@ -249,9 +252,12 @@ func TestDecodeBase64Run(t *testing.T) { "field": "correct data", } - newEvent, err := f.Run(event) + dropped, err := f.Run(ed) + assert.NoError(t, err) - assert.Equal(t, expectedFields, newEvent.Fields) - assert.Equal(t, expectedMeta, newEvent.Meta) + assert.False(t, dropped) + ed.Apply() + assert.Equal(t, expectedFields, event.Fields) + assert.Equal(t, expectedMeta, event.Meta) }) } diff --git a/libbeat/processors/actions/decode_json_fields.go b/libbeat/processors/actions/decode_json_fields.go index b47ebd646d9c..ca39dc3f4882 100644 --- a/libbeat/processors/actions/decode_json_fields.go +++ b/libbeat/processors/actions/decode_json_fields.go @@ -100,7 +100,7 @@ func NewDecodeJSONFields(c *cfg.C) (beat.Processor, error) { return f, nil } -func (f *decodeJSONFields) Run(event *beat.Event) (*beat.Event, error) { +func (f *decodeJSONFields) Run(event *beat.EventEditor) (dropped bool, err error) { var errs []string for _, field := range f.fields { @@ -122,7 +122,13 @@ func (f *decodeJSONFields) Run(event *beat.Event) (*beat.Event, error) { if err != nil { f.logger.Debugf("Error trying to unmarshal %s", text) errs = append(errs, err.Error()) - event.SetErrorWithOption(fmt.Sprintf("parsing input as JSON: %s", err.Error()), f.addErrorKey, text, field) + if f.addErrorKey { + event.AddError(beat.EventError{ + Message: fmt.Sprintf("parsing input as JSON: %s", err.Error()), + Data: text, + Field: field, + }) + } continue } @@ -169,17 +175,14 @@ func (f *decodeJSONFields) Run(event *beat.Event) (*beat.Event, error) { } if id != "" { - if event.Meta == nil { - event.Meta = mapstr.M{} - } - event.Meta[events.FieldMetaID] = id + _, _ = event.PutValue(beat.MetadataFieldKey+"."+events.FieldMetaID, id) } } if len(errs) > 0 { - return event, fmt.Errorf(strings.Join(errs, ", ")) + return false, fmt.Errorf(strings.Join(errs, ", ")) } - return event, nil + return false, nil } func unmarshal(maxDepth int, text string, fields *interface{}, processArray bool) error { diff --git a/libbeat/processors/actions/decode_json_fields_test.go b/libbeat/processors/actions/decode_json_fields_test.go index ba0e7fda347f..ae4ec43ec3e4 100644 --- a/libbeat/processors/actions/decode_json_fields_test.go +++ b/libbeat/processors/actions/decode_json_fields_test.go @@ -65,7 +65,7 @@ func TestMissingKey(t *testing.T) { "pipeline": "us1", } - actual := getActualValue(t, testConfig, input) + actual := getActualValue(t, testConfig, input, "") expected := mapstr.M{ "pipeline": "us1", @@ -80,7 +80,7 @@ func TestFieldNotString(t *testing.T) { "pipeline": "us1", } - actual := getActualValue(t, testConfig, input) + actual := getActualValue(t, testConfig, input, "") expected := mapstr.M{ "msg": 123, @@ -96,7 +96,7 @@ func TestInvalidJSON(t *testing.T) { "pipeline": "us1", } - actual := getActualValue(t, testConfig, input) + actual := getActualValue(t, testConfig, input, "unexpected EOF") expected := mapstr.M{ "msg": "{\"log\":\"{\\\"level\\\":\\\"info\\\"}\",\"stream\":\"stderr\",\"count\":3", @@ -111,7 +111,7 @@ func TestInvalidJSONMultiple(t *testing.T) { "pipeline": "us1", } - actual := getActualValue(t, testConfig, input) + actual := getActualValue(t, testConfig, input, "multiple json elements found") expected := mapstr.M{ "msg": "11:38:04,323 |-INFO testing", @@ -138,8 +138,11 @@ func TestDocumentID(t *testing.T) { t.Fatal(err) } - actual, err := p.Run(&beat.Event{Fields: input}) + event := &beat.Event{Fields: input} + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) require.NoError(t, err) + require.False(t, dropped) wantFields := mapstr.M{ "msg": map[string]interface{}{"log": "message"}, @@ -148,8 +151,9 @@ func TestDocumentID(t *testing.T) { "_id": "myDocumentID", } - assert.Equal(t, wantFields, actual.Fields) - assert.Equal(t, wantMeta, actual.Meta) + ed.Apply() + assert.Equal(t, wantFields, event.Fields) + assert.Equal(t, wantMeta, event.Meta) } func TestValidJSONDepthOne(t *testing.T) { @@ -158,7 +162,7 @@ func TestValidJSONDepthOne(t *testing.T) { "pipeline": "us1", } - actual := getActualValue(t, testConfig, input) + actual := getActualValue(t, testConfig, input, "") expected := mapstr.M{ "msg": map[string]interface{}{ @@ -184,7 +188,7 @@ func TestValidJSONDepthTwo(t *testing.T) { "max_depth": 2, }) - actual := getActualValue(t, testConfig, input) + actual := getActualValue(t, testConfig, input, "") expected := mapstr.M{ "msg": map[string]interface{}{ @@ -213,7 +217,7 @@ func TestTargetOption(t *testing.T) { "target": "doc", }) - actual := getActualValue(t, testConfig, input) + actual := getActualValue(t, testConfig, input, "") expected := mapstr.M{ "doc": map[string]interface{}{ @@ -243,7 +247,7 @@ func TestTargetRootOption(t *testing.T) { "target": "", }) - actual := getActualValue(t, testConfig, input) + actual := getActualValue(t, testConfig, input, "") expected := mapstr.M{ "log": map[string]interface{}{ @@ -282,7 +286,10 @@ func TestTargetMetadata(t *testing.T) { t.Fatal(err) } - actual, _ := p.Run(event) + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) + require.NoError(t, err) + require.False(t, dropped) expectedMeta := mapstr.M{ "json": map[string]interface{}{ @@ -293,9 +300,12 @@ func TestTargetMetadata(t *testing.T) { "count": int64(3), }, } + expectedFields := event.Fields.Clone() + + ed.Apply() - assert.Equal(t, expectedMeta, actual.Meta) - assert.Equal(t, event.Fields, actual.Fields) + assert.Equal(t, expectedMeta, event.Meta) + assert.Equal(t, expectedFields, event.Fields) } func TestNotJsonObjectOrArray(t *testing.T) { @@ -353,7 +363,7 @@ func TestNotJsonObjectOrArray(t *testing.T) { "max_depth": testCase.MaxDepth, }) - actual := getActualValue(t, testConfig, input) + actual := getActualValue(t, testConfig, input, "") assert.Equal(t, testCase.Expected.String(), actual.String()) }) } @@ -372,7 +382,7 @@ func TestArrayWithArraysDisabled(t *testing.T) { "process_array": false, }) - actual := getActualValue(t, testConfig, input) + actual := getActualValue(t, testConfig, input, "") expected := mapstr.M{ "msg": mapstr.M{ @@ -396,7 +406,7 @@ func TestArrayWithArraysEnabled(t *testing.T) { "process_array": true, }) - actual := getActualValue(t, testConfig, input) + actual := getActualValue(t, testConfig, input, "") expected := mapstr.M{ "msg": mapstr.M{ @@ -420,7 +430,7 @@ func TestArrayWithInvalidArray(t *testing.T) { "process_array": true, }) - actual := getActualValue(t, testConfig, input) + actual := getActualValue(t, testConfig, input, "") expected := mapstr.M{ "msg": mapstr.M{ @@ -438,7 +448,7 @@ func TestAddErrKeyOption(t *testing.T) { expectedOutput mapstr.M }{ {name: "With add_error_key option", addErrOption: true, expectedOutput: mapstr.M{ - "error": mapstr.M{"message": "@timestamp not overwritten (parse error on {})", "type": "json"}, + "error": mapstr.M{"message": "timestamp parse error on {}", "field": "@timestamp"}, "msg": "{\"@timestamp\":\"{}\"}", }}, {name: "Without add_error_key option", addErrOption: false, expectedOutput: mapstr.M{ @@ -457,7 +467,7 @@ func TestAddErrKeyOption(t *testing.T) { "overwrite_keys": true, "target": "", }) - actual := getActualValue(t, testConfig, input) + actual := getActualValue(t, testConfig, input, "") assert.Equal(t, test.expectedOutput.String(), actual.String()) @@ -481,7 +491,7 @@ func TestExpandKeys(t *testing.T) { }, }, } - actual := getActualValue(t, testConfig, input) + actual := getActualValue(t, testConfig, input, "") assert.Equal(t, expected, actual) } @@ -503,7 +513,7 @@ func TestExpandKeysWithTarget(t *testing.T) { }, }, } - actual := getActualValue(t, testConfig, input) + actual := getActualValue(t, testConfig, input, "") assert.Equal(t, expected, actual) } @@ -521,11 +531,10 @@ func TestExpandKeysError(t *testing.T) { "msg": `{"a.b": "c", "a.b.c": "d"}`, "error": mapstr.M{ "message": "cannot expand ...", - "type": "json", }, } - actual := getActualValue(t, testConfig, input) + actual := getActualValue(t, testConfig, input, "") assert.Contains(t, actual, "error") errorField := actual["error"].(mapstr.M) assert.Contains(t, errorField, "message") @@ -553,7 +562,7 @@ func TestOverwriteMetadata(t *testing.T) { expected := mapstr.M{ "msg": "overwrite metadata test", } - actual := getActualValue(t, testConfig, input) + actual := getActualValue(t, testConfig, input, "") assert.Equal(t, expected, actual) } @@ -568,7 +577,7 @@ func TestAddErrorToEventOnUnmarshalError(t *testing.T) { "message": "Broken JSON [[", } - actual := getActualValue(t, testConfig, input) + actual := getActualValue(t, testConfig, input, "invalid character 'B' looking for beginning of value") errObj, ok := actual["error"].(mapstr.M) require.True(t, ok, "'error' field not present or of invalid type") @@ -579,7 +588,7 @@ func TestAddErrorToEventOnUnmarshalError(t *testing.T) { assert.NotNil(t, errObj["message"]) } -func getActualValue(t *testing.T, config *conf.C, input mapstr.M) mapstr.M { +func getActualValue(t *testing.T, config *conf.C, input mapstr.M, expErr string) mapstr.M { log := logp.NewLogger("decode_json_fields_test") p, err := NewDecodeJSONFields(config) @@ -588,6 +597,16 @@ func getActualValue(t *testing.T, config *conf.C, input mapstr.M) mapstr.M { t.Fatal(err) } - actual, _ := p.Run(&beat.Event{Fields: input}) - return actual.Fields + event := &beat.Event{Fields: input} + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) + if expErr != "" { + require.Error(t, err) + require.Equal(t, expErr, err.Error()) + } else { + require.NoError(t, err) + } + require.False(t, dropped) + ed.Apply() + return event.Fields } diff --git a/libbeat/processors/actions/decompress_gzip_field.go b/libbeat/processors/actions/decompress_gzip_field.go index cdd731854206..7a8dc6a9cda3 100644 --- a/libbeat/processors/actions/decompress_gzip_field.go +++ b/libbeat/processors/actions/decompress_gzip_field.go @@ -67,28 +67,22 @@ func NewDecompressGzipFields(c *conf.C) (beat.Processor, error) { } // Run applies the decompress_gzip_fields processor to an event. -func (f *decompressGzipField) Run(event *beat.Event) (*beat.Event, error) { - var backup *beat.Event - if f.config.FailOnError { - backup = event.Clone() - } - - err := f.decompressGzipField(event) +func (f *decompressGzipField) Run(event *beat.EventEditor) (dropped bool, err error) { + err = f.decompressGzipField(event) if err != nil { errMsg := fmt.Errorf("Failed to decompress field in decompress_gzip_field processor: %w", err) if publisher.LogWithTrace() { f.log.Debug(errMsg.Error()) } if f.config.FailOnError { - event = backup - _, _ = event.PutValue("error.message", errMsg.Error()) - return event, err + event.AddError(beat.EventError{Message: errMsg.Error()}) + return false, err } } - return event, nil + return false, nil } -func (f *decompressGzipField) decompressGzipField(event *beat.Event) error { +func (f *decompressGzipField) decompressGzipField(event *beat.EventEditor) error { data, err := event.GetValue(f.config.Field.From) if err != nil { if f.config.IgnoreMissing && errors.Is(err, mapstr.ErrKeyNotFound) { diff --git a/libbeat/processors/actions/decompress_gzip_field_test.go b/libbeat/processors/actions/decompress_gzip_field_test.go index 230e36ecb6f5..49f89bf830be 100644 --- a/libbeat/processors/actions/decompress_gzip_field_test.go +++ b/libbeat/processors/actions/decompress_gzip_field_test.go @@ -178,14 +178,16 @@ func TestDecompressGzip(t *testing.T) { Fields: test.input, } - newEvent, err := f.Run(event) + ed := beat.NewEventEditor(event) + dropped, err := f.Run(ed) if !test.error { assert.NoError(t, err) } else { assert.Error(t, err) } - - assert.Equal(t, test.output, newEvent.Fields) + assert.False(t, dropped) + ed.Apply() + assert.Equal(t, test.output, event.Fields) }) } @@ -202,6 +204,7 @@ func TestDecompressGzip(t *testing.T) { expectedMeta := mapstr.M{ "field": "decompressed data", } + expectedFields := event.Fields.Clone() config := decompressGzipFieldConfig{ Field: fromTo{ @@ -216,10 +219,13 @@ func TestDecompressGzip(t *testing.T) { config: config, } - newEvent, err := f.Run(event) + ed := beat.NewEventEditor(event) + dropped, err := f.Run(ed) assert.NoError(t, err) + assert.False(t, dropped) - assert.Equal(t, expectedMeta, newEvent.Meta) - assert.Equal(t, event.Fields, newEvent.Fields) + ed.Apply() + assert.Equal(t, expectedMeta, event.Meta) + assert.Equal(t, expectedFields, event.Fields) }) } diff --git a/libbeat/processors/actions/detect_mime_type.go b/libbeat/processors/actions/detect_mime_type.go index 5a5ecf34d2f1..21980808f90e 100644 --- a/libbeat/processors/actions/detect_mime_type.go +++ b/libbeat/processors/actions/detect_mime_type.go @@ -49,21 +49,21 @@ func NewDetectMimeType(cfg *conf.C) (beat.Processor, error) { return mimeType, nil } -func (m *mimeTypeProcessor) Run(event *beat.Event) (*beat.Event, error) { +func (m *mimeTypeProcessor) Run(event *beat.EventEditor) (dropped bool, err error) { valI, err := event.GetValue(m.Field) if err != nil { //nolint:nilerr // doesn't have the required field value to analyze - return event, nil + return false, nil } val, _ := valI.(string) if val == "" { // wrong type or not set - return event, nil + return false, nil } if mimeType := mime.Detect(val); mimeType != "" { _, err = event.PutValue(m.Target, mimeType) } - return event, err + return false, err } func (m *mimeTypeProcessor) String() string { diff --git a/libbeat/processors/actions/detect_mime_type_test.go b/libbeat/processors/actions/detect_mime_type_test.go index 3a8a60aa282d..1272efa33fbe 100644 --- a/libbeat/processors/actions/detect_mime_type_test.go +++ b/libbeat/processors/actions/detect_mime_type_test.go @@ -28,7 +28,7 @@ import ( ) func TestMimeTypeFromTo(t *testing.T) { - evt := beat.Event{ + evt := &beat.Event{ Fields: mapstr.M{ "foo.bar.baz": "hello world!", }, @@ -38,15 +38,20 @@ func TestMimeTypeFromTo(t *testing.T) { "target": "bar.baz.zoiks", })) require.NoError(t, err) - observed, err := p.Run(&evt) + + ed := beat.NewEventEditor(evt) + dropped, err := p.Run(ed) require.NoError(t, err) - enriched, err := observed.Fields.GetValue("bar.baz.zoiks") + require.False(t, dropped) + + ed.Apply() + enriched, err := evt.Fields.GetValue("bar.baz.zoiks") require.NoError(t, err) require.Equal(t, "text/plain; charset=utf-8", enriched) } func TestMimeTypeFromToMetadata(t *testing.T) { - evt := beat.Event{ + evt := &beat.Event{ Meta: mapstr.M{}, Fields: mapstr.M{ "foo.bar.baz": "hello world!", @@ -55,20 +60,25 @@ func TestMimeTypeFromToMetadata(t *testing.T) { expectedMeta := mapstr.M{ "field": "text/plain; charset=utf-8", } + expectedFields := evt.Fields.Clone() + p, err := NewDetectMimeType(conf.MustNewConfigFrom(map[string]interface{}{ "field": "foo.bar.baz", "target": "@metadata.field", })) require.NoError(t, err) - observed, err := p.Run(&evt) + ed := beat.NewEventEditor(evt) + dropped, err := p.Run(ed) require.NoError(t, err) - require.Equal(t, expectedMeta, observed.Meta) - require.Equal(t, evt.Fields, observed.Fields) + require.False(t, dropped) + ed.Apply() + require.Equal(t, expectedMeta, evt.Meta) + require.Equal(t, expectedFields, evt.Fields) } func TestMimeTypeTestNoMatch(t *testing.T) { - evt := beat.Event{ + evt := &beat.Event{ Fields: mapstr.M{ "foo.bar.baz": string([]byte{0, 0}), }, @@ -78,8 +88,13 @@ func TestMimeTypeTestNoMatch(t *testing.T) { "target": "bar.baz.zoiks", })) require.NoError(t, err) - observed, err := p.Run(&evt) + + ed := beat.NewEventEditor(evt) + dropped, err := p.Run(ed) require.NoError(t, err) - hasKey, _ := observed.Fields.HasKey("bar.baz.zoiks") + require.False(t, dropped) + ed.Apply() + + hasKey, _ := evt.Fields.HasKey("bar.baz.zoiks") require.False(t, hasKey) } diff --git a/libbeat/processors/actions/drop_event.go b/libbeat/processors/actions/drop_event.go index bd0ef756b081..f9060ac3c92e 100644 --- a/libbeat/processors/actions/drop_event.go +++ b/libbeat/processors/actions/drop_event.go @@ -37,9 +37,8 @@ func newDropEvent(c *conf.C) (beat.Processor, error) { return dropEventsSingleton, nil } -func (*dropEvent) Run(_ *beat.Event) (*beat.Event, error) { - // return event=nil to delete the entire event - return nil, nil +func (*dropEvent) Run(*beat.EventEditor) (dropped bool, err error) { + return true, nil } func (*dropEvent) String() string { return "drop_event" } diff --git a/libbeat/processors/actions/drop_fields.go b/libbeat/processors/actions/drop_fields.go index a90d6438125b..336344a5099f 100644 --- a/libbeat/processors/actions/drop_fields.go +++ b/libbeat/processors/actions/drop_fields.go @@ -91,27 +91,51 @@ func newDropFields(c *conf.C) (beat.Processor, error) { return f, nil } -func (f *dropFields) Run(event *beat.Event) (*beat.Event, error) { +func (f *dropFields) Run(event *beat.EventEditor) (dropped bool, err error) { var errs []error + droppedKeys := make(map[string]struct{}) // remove exact match fields for _, field := range f.Fields { + if f.checkAlreadyDropped(droppedKeys, field) { + continue + } + droppedKeys[field] = struct{}{} f.deleteField(event, field, &errs) } // remove fields contained in regexp expressions - for _, regex := range f.RegexpFields { - for _, field := range *event.Fields.FlattenKeys() { - if regex.MatchString(field) { - f.deleteField(event, field, &errs) + for _, field := range event.FlattenKeys() { + if f.checkAlreadyDropped(droppedKeys, field) { + continue + } + for _, regex := range f.RegexpFields { + if !regex.MatchString(field) { + continue } + droppedKeys[field] = struct{}{} + f.deleteField(event, field, &errs) + } + } + + return false, multierr.Combine(errs...) +} + +func (f *dropFields) checkAlreadyDropped(droppedKeys map[string]struct{}, key string) bool { + _, dropped := droppedKeys[key] + if dropped { + return true + } + for droppedKey := range droppedKeys { + if strings.HasPrefix(key, droppedKey) { + return true } } - return event, multierr.Combine(errs...) + return false } -func (f *dropFields) deleteField(event *beat.Event, field string, errs *[]error) { +func (f *dropFields) deleteField(event *beat.EventEditor, field string, errs *[]error) { if err := event.Delete(field); err != nil { if !f.IgnoreMissing || !errors.Is(err, mapstr.ErrKeyNotFound) { *errs = append(*errs, fmt.Errorf("failed to drop field [%v], error: %w", field, err)) diff --git a/libbeat/processors/actions/drop_fields_test.go b/libbeat/processors/actions/drop_fields_test.go index f8a701d5af07..8c580fbd1969 100644 --- a/libbeat/processors/actions/drop_fields_test.go +++ b/libbeat/processors/actions/drop_fields_test.go @@ -44,10 +44,18 @@ func TestDropFieldRun(t *testing.T) { Fields: []string{"field"}, } - newEvent, err := p.Run(event) + cloned := &beat.Event{ + Fields: event.Fields.Clone(), + Meta: event.Meta.Clone(), + } + ed := beat.NewEventEditor(cloned) + + dropped, err := p.Run(ed) assert.NoError(t, err) - assert.Equal(t, mapstr.M{}, newEvent.Fields) - assert.Equal(t, event.Meta, newEvent.Meta) + assert.False(t, dropped) + ed.Apply() + assert.Equal(t, mapstr.M{}, cloned.Fields) + assert.Equal(t, event.Meta, cloned.Meta) }) t.Run("supports a metadata field", func(t *testing.T) { @@ -55,10 +63,18 @@ func TestDropFieldRun(t *testing.T) { Fields: []string{"@metadata.meta_field"}, } - newEvent, err := p.Run(event) + cloned := &beat.Event{ + Fields: event.Fields.Clone(), + Meta: event.Meta.Clone(), + } + ed := beat.NewEventEditor(cloned) + + dropped, err := p.Run(ed) assert.NoError(t, err) - assert.Equal(t, mapstr.M{}, newEvent.Meta) - assert.Equal(t, event.Fields, newEvent.Fields) + assert.False(t, dropped) + ed.Apply() + assert.Equal(t, mapstr.M{}, cloned.Meta) + assert.Equal(t, event.Fields, cloned.Fields) }) t.Run("supports a regexp field", func(t *testing.T) { @@ -80,17 +96,28 @@ func TestDropFieldRun(t *testing.T) { } p := dropFields{ - RegexpFields: []match.Matcher{match.MustCompile("field_2$"), match.MustCompile("field_1\\.(.*)\\.subfield_2_1"), match.MustCompile("field_1\\.subfield_3(.*)")}, - Fields: []string{}, + RegexpFields: []match.Matcher{ + match.MustCompile("field_2$"), + match.MustCompile("field_1\\.(.*)\\.subfield_2_1"), + match.MustCompile("field_1\\.subfield_3(.*)"), + }, + Fields: []string{}, } - newEvent, err := p.Run(event) - assert.NoError(t, err) - assert.Equal(t, mapstr.M{ + expFields := mapstr.M{ "field_1": mapstr.M{ "subfield_1": "sf_1_value", }, - }, newEvent.Fields) + } + + ed := beat.NewEventEditor(event) + + dropped, err := p.Run(ed) + assert.NoError(t, err) + assert.False(t, dropped) + ed.Apply() + + assert.Equal(t, expFields, event.Fields) }) } diff --git a/libbeat/processors/actions/extract_field.go b/libbeat/processors/actions/extract_field.go index 0146526939d7..04b949207bbf 100644 --- a/libbeat/processors/actions/extract_field.go +++ b/libbeat/processors/actions/extract_field.go @@ -72,26 +72,26 @@ func NewExtractField(c *conf.C) (beat.Processor, error) { return f, nil } -func (f *extract_field) Run(event *beat.Event) (*beat.Event, error) { +func (f *extract_field) Run(event *beat.EventEditor) (dropped bool, err error) { fieldValue, err := event.GetValue(f.Field) if err != nil { - return event, fmt.Errorf("error getting field '%s' from event", f.Field) + return false, fmt.Errorf("error getting field '%s' from event", f.Field) } value, ok := fieldValue.(string) if !ok { - return event, fmt.Errorf("could not get a string from field '%s'", f.Field) + return false, fmt.Errorf("could not get a string from field '%s'", f.Field) } parts := strings.Split(value, f.Separator) parts = deleteEmpty(parts) if len(parts) < f.Index+1 { - return event, fmt.Errorf("index is out of range for field '%s'", f.Field) + return false, fmt.Errorf("index is out of range for field '%s'", f.Field) } _, _ = event.PutValue(f.Target, parts[f.Index]) - return event, nil + return false, nil } func (f extract_field) String() string { diff --git a/libbeat/processors/actions/extract_field_test.go b/libbeat/processors/actions/extract_field_test.go index 2b71105c65b4..d7c58c4723a4 100644 --- a/libbeat/processors/actions/extract_field_test.go +++ b/libbeat/processors/actions/extract_field_test.go @@ -91,21 +91,18 @@ func TestCommonPaths(t *testing.T) { test.Field: test.Value, } - event, err := runExtractField(t, testConfig, input) + event, dropped, err := runExtractField(t, testConfig, input) if test.Error { assert.Error(t, err) } else { - assert.NoError(t, err) - result, err := event.Fields.GetValue(test.Target) + result, err := event.GetValue(test.Target) if err != nil { t.Fatalf("could not get target field: %s", err) } assert.Equal(t, result.(string), test.Result) } - - // Event must be present, even on error - assert.NotNil(t, event) + assert.False(t, dropped) } t.Run("supports a metadata field", func(t *testing.T) { @@ -135,14 +132,17 @@ func TestCommonPaths(t *testing.T) { t.Fatalf("error initializing extract_field: %s", err) } - newEvent, err := p.Run(event) + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) assert.NoError(t, err) - assert.Equal(t, expectedFields, newEvent.Fields) - assert.Equal(t, expectedMeta, newEvent.Meta) + assert.False(t, dropped) + ed.Apply() + assert.Equal(t, expectedFields, event.Fields) + assert.Equal(t, expectedMeta, event.Meta) }) } -func runExtractField(t *testing.T, config *conf.C, input mapstr.M) (*beat.Event, error) { +func runExtractField(t *testing.T, config *conf.C, input mapstr.M) (event *beat.Event, dropped bool, err error) { logp.TestingSetup() p, err := NewExtractField(config) @@ -150,5 +150,9 @@ func runExtractField(t *testing.T, config *conf.C, input mapstr.M) (*beat.Event, t.Fatalf("error initializing extract_field: %s", err) } - return p.Run(&beat.Event{Fields: input}) + event = &beat.Event{Fields: input} + ed := beat.NewEventEditor(event) + dropped, err = p.Run(ed) + ed.Apply() + return event, dropped, err } diff --git a/libbeat/processors/actions/include_fields.go b/libbeat/processors/actions/include_fields.go index 08419e7c2eca..83f30ade596d 100644 --- a/libbeat/processors/actions/include_fields.go +++ b/libbeat/processors/actions/include_fields.go @@ -66,7 +66,7 @@ func newIncludeFields(c *conf.C) (beat.Processor, error) { return f, nil } -func (f *includeFields) Run(event *beat.Event) (*beat.Event, error) { +func (f *includeFields) Run(event *beat.EventEditor) (dropped bool, err error) { filtered := mapstr.M{} var errs []string @@ -82,11 +82,15 @@ func (f *includeFields) Run(event *beat.Event) (*beat.Event, error) { } } - event.Fields = filtered + // does not apply the changes yet, only marks + event.DeleteAll() + for key := range filtered { + _, _ = event.PutValue(key, filtered[key]) + } if len(errs) > 0 { - return event, fmt.Errorf(strings.Join(errs, ", ")) + return false, fmt.Errorf(strings.Join(errs, ", ")) } - return event, nil + return false, nil } func (f *includeFields) String() string { diff --git a/libbeat/processors/actions/include_fields_test.go b/libbeat/processors/actions/include_fields_test.go index 9e5e6199feca..fbcb278aedc4 100644 --- a/libbeat/processors/actions/include_fields_test.go +++ b/libbeat/processors/actions/include_fields_test.go @@ -68,9 +68,11 @@ func TestIncludeFields(t *testing.T) { Fields: test.Input, } - newEvent, err := p.Run(event) + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) assert.NoError(t, err) - - assert.Equal(t, test.Output, newEvent.Fields) + assert.False(t, dropped) + ed.Apply() + assert.Equal(t, test.Output, event.Fields) } } diff --git a/libbeat/processors/actions/rename.go b/libbeat/processors/actions/rename.go index 15d2627540bf..82909c5dd915 100644 --- a/libbeat/processors/actions/rename.go +++ b/libbeat/processors/actions/rename.go @@ -73,13 +73,7 @@ func NewRenameFields(c *conf.C) (beat.Processor, error) { return f, nil } -func (f *renameFields) Run(event *beat.Event) (*beat.Event, error) { - var backup *beat.Event - // Creates a copy of the event to revert in case of failure - if f.config.FailOnError { - backup = event.Clone() - } - +func (f *renameFields) Run(event *beat.EventEditor) (dropped bool, err error) { for _, field := range f.config.Fields { err := f.renameField(field.From, field.To, event) if err != nil { @@ -88,17 +82,15 @@ func (f *renameFields) Run(event *beat.Event) (*beat.Event, error) { f.logger.Debug(errMsg.Error()) } if f.config.FailOnError { - event = backup - _, _ = event.PutValue("error.message", errMsg.Error()) - return event, err + return false, beat.EventError{Message: errMsg.Error(), Processor: f.String()} } } } - return event, nil + return false, nil } -func (f *renameFields) renameField(from string, to string, event *beat.Event) error { +func (f *renameFields) renameField(from string, to string, event *beat.EventEditor) error { // Fields cannot be overwritten. Either the target field has to be dropped first or renamed first _, err := event.GetValue(to) if err == nil { diff --git a/libbeat/processors/actions/rename_test.go b/libbeat/processors/actions/rename_test.go index 05f66eaf2ef1..ab746f058212 100644 --- a/libbeat/processors/actions/rename_test.go +++ b/libbeat/processors/actions/rename_test.go @@ -18,7 +18,6 @@ package actions import ( - "reflect" "testing" "github.com/elastic/elastic-agent-libs/logp" @@ -39,6 +38,7 @@ func TestRenameRun(t *testing.T) { Input mapstr.M Output mapstr.M error bool + expErrMsg string }{ { description: "simple field renaming", @@ -95,11 +95,9 @@ func TestRenameRun(t *testing.T) { Output: mapstr.M{ "a": 2, "b": "q", - "error": mapstr.M{ - "message": "Failed to rename fields in processor: target field b already exists, drop or rename this field first", - }, }, error: true, + expErrMsg: "[processor=rename=[{From:a To:b}]] Failed to rename fields in processor: target field b already exists, drop or rename this field first", FailOnError: true, IgnoreMissing: false, }, @@ -194,11 +192,9 @@ func TestRenameRun(t *testing.T) { Output: mapstr.M{ "a": 9, "c": 10, - "error": mapstr.M{ - "message": "Failed to rename fields in processor: could not put value: a.c: 10, expected map but type is int", - }, }, error: true, + expErrMsg: "[processor=rename=[{From:c To:a.c} {From:a To:a.value}]] Failed to rename fields in processor: could not put value: a.c: 10, expected map but type is int", IgnoreMissing: false, FailOnError: true, }, @@ -240,17 +236,25 @@ func TestRenameRun(t *testing.T) { logger: log, } event := &beat.Event{ - Fields: test.Input, + Meta: mapstr.M{}, + Fields: test.Input.Clone(), } - newEvent, err := f.Run(event) + ed := beat.NewEventEditor(event) + dropped, err := f.Run(ed) if !test.error { assert.NoError(t, err) + ed.Apply() } else { assert.Error(t, err) + if test.expErrMsg != "" { + assert.Equal(t, test.expErrMsg, err.Error()) + } + ed.Reset() } - - assert.True(t, reflect.DeepEqual(newEvent.Fields, test.Output)) + assert.False(t, dropped) + ed.Apply() + assert.Equal(t, test.Output, event.Fields) }) } } @@ -281,7 +285,7 @@ func TestRenameField(t *testing.T) { ignoreMissing: false, }, { - description: "Add hierarchy to event", + description: "add hierarchy to event", From: "a.b", To: "a.b.c", Input: mapstr.M{ @@ -358,17 +362,21 @@ func TestRenameField(t *testing.T) { }, } - err := f.renameField(test.From, test.To, &beat.Event{Fields: test.Input}) + event := &beat.Event{Fields: test.Input.Clone()} + ed := beat.NewEventEditor(event) + err := f.renameField(test.From, test.To, ed) if err != nil { assert.Equal(t, test.error, true) } - assert.True(t, reflect.DeepEqual(test.Input, test.Output)) + ed.Apply() + assert.Equal(t, test.Output, event.Fields) }) } t.Run("supports metadata as a target", func(t *testing.T) { event := &beat.Event{ + Fields: mapstr.M{}, Meta: mapstr.M{ "a": "c", }, @@ -378,6 +386,8 @@ func TestRenameField(t *testing.T) { "b": "c", } + expFields := event.Fields.Clone() + f := &renameFields{ config: renameFieldsConfig{ Fields: []fromTo{ @@ -389,9 +399,12 @@ func TestRenameField(t *testing.T) { }, } - newEvent, err := f.Run(event) + ed := beat.NewEventEditor(event) + dropped, err := f.Run(ed) assert.NoError(t, err) - assert.Equal(t, expMeta, newEvent.Meta) - assert.Equal(t, event.Fields, newEvent.Fields) + assert.False(t, dropped) + ed.Apply() + assert.Equal(t, expMeta, event.Meta) + assert.Equal(t, expFields, event.Fields) }) } diff --git a/libbeat/processors/actions/replace.go b/libbeat/processors/actions/replace.go index 07e5ba0fcc00..ef98477151fe 100644 --- a/libbeat/processors/actions/replace.go +++ b/libbeat/processors/actions/replace.go @@ -75,13 +75,7 @@ func NewReplaceString(c *conf.C) (beat.Processor, error) { return f, nil } -func (f *replaceString) Run(event *beat.Event) (*beat.Event, error) { - var backup *beat.Event - // Creates a copy of the event to revert in case of failure - if f.config.FailOnError { - backup = event.Clone() - } - +func (f *replaceString) Run(event *beat.EventEditor) (dropped bool, err error) { for _, field := range f.config.Fields { err := f.replaceField(field.Field, field.Pattern, field.Replacement, event) if err != nil { @@ -90,17 +84,15 @@ func (f *replaceString) Run(event *beat.Event) (*beat.Event, error) { f.log.Debug(errMsg.Error()) } if f.config.FailOnError { - event = backup - _, _ = event.PutValue("error.message", errMsg.Error()) - return event, err + return false, beat.EventError{Message: errMsg.Error(), Processor: f.String()} } } } - return event, nil + return false, nil } -func (f *replaceString) replaceField(field string, pattern *regexp.Regexp, replacement string, event *beat.Event) error { +func (f *replaceString) replaceField(field string, pattern *regexp.Regexp, replacement string, event *beat.EventEditor) error { currentValue, err := event.GetValue(field) if err != nil { // Ignore ErrKeyNotFound errors diff --git a/libbeat/processors/actions/replace_test.go b/libbeat/processors/actions/replace_test.go index 53977d39b92b..2f9c80563be4 100644 --- a/libbeat/processors/actions/replace_test.go +++ b/libbeat/processors/actions/replace_test.go @@ -18,7 +18,6 @@ package actions import ( - "reflect" "regexp" "testing" @@ -39,6 +38,7 @@ func TestReplaceRun(t *testing.T) { Input mapstr.M Output mapstr.M error bool + expErrMsg string }{ { description: "simple field replacing", @@ -129,11 +129,9 @@ func TestReplaceRun(t *testing.T) { Output: mapstr.M{ "m": "abc", "n": "def", - "error": mapstr.M{ - "message": "Failed to replace fields in processor: could not fetch value for key: f, Error: key not found", - }, }, error: true, + expErrMsg: "[processor=replace=[{Field:f Pattern:abc Replacement:xyz} {Field:g Pattern:def Replacement:}]] Failed to replace fields in processor: could not fetch value for key: f, Error: key not found", IgnoreMissing: false, FailOnError: true, }, @@ -153,14 +151,21 @@ func TestReplaceRun(t *testing.T) { Fields: test.Input, } - newEvent, err := f.Run(event) + ed := beat.NewEventEditor(event) + dropped, err := f.Run(ed) if !test.error { assert.NoError(t, err) + ed.Apply() } else { assert.Error(t, err) + if test.expErrMsg != "" { + assert.Equal(t, test.expErrMsg, err.Error()) + } + ed.Reset() } - - assert.True(t, reflect.DeepEqual(newEvent.Fields, test.Output)) + assert.False(t, dropped) + ed.Apply() + assert.Equal(t, test.Output, event.Fields) }) } } @@ -240,17 +245,20 @@ func TestReplaceField(t *testing.T) { }, } - err := f.replaceField(test.Field, test.Pattern, test.Replacement, &beat.Event{Fields: test.Input}) + event := &beat.Event{Fields: test.Input.Clone()} + ed := beat.NewEventEditor(event) + err := f.replaceField(test.Field, test.Pattern, test.Replacement, ed) if err != nil { assert.Equal(t, test.error, true) } - - assert.True(t, reflect.DeepEqual(test.Input, test.Output)) + ed.Apply() + assert.Equal(t, test.Output, event.Fields) }) } t.Run("supports metadata as a target", func(t *testing.T) { event := &beat.Event{ + Fields: mapstr.M{}, Meta: mapstr.M{ "f": "abc", }, @@ -260,6 +268,8 @@ func TestReplaceField(t *testing.T) { "f": "bbc", } + expectedFields := event.Fields.Clone() + f := &replaceString{ config: replaceStringConfig{ Fields: []replaceConfig{ @@ -272,9 +282,12 @@ func TestReplaceField(t *testing.T) { }, } - newEvent, err := f.Run(event) + ed := beat.NewEventEditor(event) + dropped, err := f.Run(ed) assert.NoError(t, err) - assert.Equal(t, expectedMeta, newEvent.Meta) - assert.Equal(t, event.Fields, newEvent.Fields) + assert.False(t, dropped) + ed.Apply() + assert.Equal(t, expectedMeta, event.Meta) + assert.Equal(t, expectedFields, event.Fields) }) } diff --git a/libbeat/processors/actions/truncate_fields.go b/libbeat/processors/actions/truncate_fields.go index 3eab824c99b5..7688a85f0545 100644 --- a/libbeat/processors/actions/truncate_fields.go +++ b/libbeat/processors/actions/truncate_fields.go @@ -81,27 +81,22 @@ func NewTruncateFields(c *conf.C) (beat.Processor, error) { }, nil } -func (f *truncateFields) Run(event *beat.Event) (*beat.Event, error) { - var backup *beat.Event - if f.config.FailOnError { - backup = event.Clone() - } - +func (f *truncateFields) Run(event *beat.EventEditor) (dropped bool, err error) { for _, field := range f.config.Fields { err := f.truncateSingleField(field, event) if err != nil { - f.logger.Debugf("Failed to truncate fields: %s", err) + errMsg := fmt.Errorf("Failed to truncate field: %s", err) + f.logger.Debugf(errMsg.Error()) if f.config.FailOnError { - event = backup - return event, err + return false, beat.EventError{Message: errMsg.Error(), Field: field, Processor: f.String()} } } } - return event, nil + return false, nil } -func (f *truncateFields) truncateSingleField(field string, event *beat.Event) error { +func (f *truncateFields) truncateSingleField(field string, event *beat.EventEditor) error { v, err := event.GetValue(field) if err != nil { if f.config.IgnoreMissing && errors.Is(err, mapstr.ErrKeyNotFound) { @@ -121,7 +116,7 @@ func (f *truncateFields) truncateSingleField(field string, event *beat.Event) er } -func (f *truncateFields) addTruncatedString(field, value string, event *beat.Event) error { +func (f *truncateFields) addTruncatedString(field, value string, event *beat.EventEditor) error { truncated, isTruncated, err := f.truncate(f, []byte(value)) if err != nil { return err @@ -132,12 +127,12 @@ func (f *truncateFields) addTruncatedString(field, value string, event *beat.Eve } if isTruncated { - _ = mapstr.AddTagsWithKey(event.Fields, "log.flags", []string{"truncated"}) + _ = event.AddTagsWithKey("log.flags", "truncated") } return nil } -func (f *truncateFields) addTruncatedByte(field string, value []byte, event *beat.Event) error { +func (f *truncateFields) addTruncatedByte(field string, value []byte, event *beat.EventEditor) error { truncated, isTruncated, err := f.truncate(f, value) if err != nil { return err @@ -148,7 +143,7 @@ func (f *truncateFields) addTruncatedByte(field string, value []byte, event *bea } if isTruncated { - _ = mapstr.AddTagsWithKey(event.Fields, "log.flags", []string{"truncated"}) + _ = event.AddTagsWithKey("log.flags", "truncated") } return nil } diff --git a/libbeat/processors/actions/truncate_fields_test.go b/libbeat/processors/actions/truncate_fields_test.go index 72d40649c084..a8213f790757 100644 --- a/libbeat/processors/actions/truncate_fields_test.go +++ b/libbeat/processors/actions/truncate_fields_test.go @@ -36,6 +36,7 @@ func TestTruncateFields(t *testing.T) { Input mapstr.M Output mapstr.M ShouldError bool + ExpErrMsg string TruncateFunc truncater }{ "truncate bytes of too long string line": { @@ -97,6 +98,7 @@ func TestTruncateFields(t *testing.T) { "message": 42, }, ShouldError: true, + ExpErrMsg: `[processor=truncate_fields=messagemax_bytes=5] [field="message"] Failed to truncate field: value cannot be truncated: 42`, TruncateFunc: (*truncateFields).truncateBytes, }, "do not truncate characters of short byte line": { @@ -168,14 +170,21 @@ func TestTruncateFields(t *testing.T) { Fields: test.Input, } - newEvent, err := p.Run(event) + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) if test.ShouldError { assert.Error(t, err) + if test.ExpErrMsg != "" { + assert.Equal(t, test.ExpErrMsg, err.Error()) + } + ed.Reset() } else { assert.NoError(t, err) + ed.Apply() } - - assert.Equal(t, test.Output, newEvent.Fields) + assert.False(t, dropped) + ed.Apply() + assert.Equal(t, test.Output, event.Fields) }) } @@ -207,10 +216,13 @@ func TestTruncateFields(t *testing.T) { "message": "too", } - newEvent, err := p.Run(event) + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) assert.NoError(t, err) + assert.False(t, dropped) - assert.Equal(t, expFields, newEvent.Fields) - assert.Equal(t, expMeta, newEvent.Meta) + ed.Apply() + assert.Equal(t, expFields, event.Fields) + assert.Equal(t, expMeta, event.Meta) }) } diff --git a/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go b/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go index 116c561a5097..972c9bc3ff4f 100644 --- a/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go +++ b/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go @@ -111,24 +111,24 @@ func (p *addCloudMetadata) getMeta() mapstr.M { return p.metadata.Clone() } -func (p *addCloudMetadata) Run(event *beat.Event) (*beat.Event, error) { +func (p *addCloudMetadata) Run(event *beat.EventEditor) (dropped bool, err error) { meta := p.getMeta() if len(meta) == 0 { - return event, nil + return false, nil } - err := p.addMeta(event, meta) + err = p.addMeta(event, meta) if err != nil { - return nil, err + return true, err } - return event, err + return false, err } func (p *addCloudMetadata) String() string { return "add_cloud_metadata=" + p.getMeta().String() } -func (p *addCloudMetadata) addMeta(event *beat.Event, meta mapstr.M) error { +func (p *addCloudMetadata) addMeta(event *beat.EventEditor, meta mapstr.M) error { for key, metaVal := range meta { // If key exists in event already and overwrite flag is set to false, this processor will not overwrite the // meta fields. For example aws module writes cloud.instance.* to events already, with overwrite=false, diff --git a/libbeat/processors/add_cloud_metadata/provider_alibaba_cloud_test.go b/libbeat/processors/add_cloud_metadata/provider_alibaba_cloud_test.go index 7ad595ab8e91..ae3f31a048ec 100644 --- a/libbeat/processors/add_cloud_metadata/provider_alibaba_cloud_test.go +++ b/libbeat/processors/add_cloud_metadata/provider_alibaba_cloud_test.go @@ -69,10 +69,11 @@ func TestRetrieveAlibabaCloudMetadata(t *testing.T) { t.Fatal(err) } - actual, err := p.Run(&beat.Event{Fields: mapstr.M{}}) - if err != nil { - t.Fatal(err) - } + event := &beat.Event{Fields: mapstr.M{}} + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) + assert.NoError(t, err) + assert.False(t, dropped) expected := mapstr.M{ "cloud": mapstr.M{ @@ -87,5 +88,6 @@ func TestRetrieveAlibabaCloudMetadata(t *testing.T) { }, }, } - assert.Equal(t, expected, actual.Fields) + ed.Apply() + assert.Equal(t, expected, event.Fields) } diff --git a/libbeat/processors/add_cloud_metadata/provider_aws_ec2_test.go b/libbeat/processors/add_cloud_metadata/provider_aws_ec2_test.go index 92e3ae7ec2e3..3cb14f27d9cd 100644 --- a/libbeat/processors/add_cloud_metadata/provider_aws_ec2_test.go +++ b/libbeat/processors/add_cloud_metadata/provider_aws_ec2_test.go @@ -359,11 +359,14 @@ func TestRetrieveAWSMetadataEC2(t *testing.T) { t.Fatalf("error creating new metadata processor: %s", err.Error()) } - actual, err := cmp.Run(&beat.Event{Fields: tc.previousEvent}) - if err != nil { - t.Fatalf("error running processor: %s", err.Error()) - } - assert.Equal(t, tc.expectedEvent, actual.Fields) + event := &beat.Event{Fields: tc.previousEvent} + ed := beat.NewEventEditor(event) + dropped, err := cmp.Run(ed) + assert.NoError(t, err) + assert.False(t, dropped) + + ed.Apply() + assert.Equal(t, tc.expectedEvent, event.Fields) }) } } diff --git a/libbeat/processors/add_cloud_metadata/provider_azure_vm_test.go b/libbeat/processors/add_cloud_metadata/provider_azure_vm_test.go index 30b663bdfc54..439ba290fe5d 100644 --- a/libbeat/processors/add_cloud_metadata/provider_azure_vm_test.go +++ b/libbeat/processors/add_cloud_metadata/provider_azure_vm_test.go @@ -105,10 +105,11 @@ func TestRetrieveAzureMetadata(t *testing.T) { t.Fatal(err) } - actual, err := p.Run(&beat.Event{Fields: mapstr.M{}}) - if err != nil { - t.Fatal(err) - } + event := &beat.Event{Fields: mapstr.M{}} + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) + assert.NoError(t, err) + assert.False(t, dropped) expected := mapstr.M{ "cloud": mapstr.M{ @@ -129,5 +130,6 @@ func TestRetrieveAzureMetadata(t *testing.T) { "region": "eastus", }, } - assert.Equal(t, expected, actual.Fields) + ed.Apply() + assert.Equal(t, expected, event.Fields) } diff --git a/libbeat/processors/add_cloud_metadata/provider_digital_ocean_test.go b/libbeat/processors/add_cloud_metadata/provider_digital_ocean_test.go index 3160d8eb82d9..f149673fe3a9 100644 --- a/libbeat/processors/add_cloud_metadata/provider_digital_ocean_test.go +++ b/libbeat/processors/add_cloud_metadata/provider_digital_ocean_test.go @@ -107,10 +107,11 @@ func TestRetrieveDigitalOceanMetadata(t *testing.T) { t.Fatal(err) } - actual, err := p.Run(&beat.Event{Fields: mapstr.M{}}) - if err != nil { - t.Fatal(err) - } + event := &beat.Event{Fields: mapstr.M{}} + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) + assert.NoError(t, err) + assert.False(t, dropped) expected := mapstr.M{ "cloud": mapstr.M{ @@ -124,5 +125,6 @@ func TestRetrieveDigitalOceanMetadata(t *testing.T) { "region": "nyc3", }, } - assert.Equal(t, expected, actual.Fields) + ed.Apply() + assert.Equal(t, expected, event.Fields) } diff --git a/libbeat/processors/add_cloud_metadata/provider_google_gce_test.go b/libbeat/processors/add_cloud_metadata/provider_google_gce_test.go index 97078f9a2802..c24ad67e0ad2 100644 --- a/libbeat/processors/add_cloud_metadata/provider_google_gce_test.go +++ b/libbeat/processors/add_cloud_metadata/provider_google_gce_test.go @@ -325,10 +325,11 @@ func TestRetrieveGCEMetadata(t *testing.T) { t.Fatal(err) } - actual, err := p.Run(&beat.Event{Fields: mapstr.M{}}) - if err != nil { - t.Fatal(err) - } + event := &beat.Event{Fields: mapstr.M{}} + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) + assert.NoError(t, err) + assert.False(t, dropped) expected := mapstr.M{ "cloud": mapstr.M{ @@ -353,7 +354,8 @@ func TestRetrieveGCEMetadata(t *testing.T) { }, }, } - assert.Equal(t, expected, actual.Fields) + ed.Apply() + assert.Equal(t, expected, event.Fields) } func TestRetrieveGCEMetadataInK8s(t *testing.T) { @@ -374,10 +376,11 @@ func TestRetrieveGCEMetadataInK8s(t *testing.T) { t.Fatal(err) } - actual, err := p.Run(&beat.Event{Fields: mapstr.M{}}) - if err != nil { - t.Fatal(err) - } + event := &beat.Event{Fields: mapstr.M{}} + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) + assert.NoError(t, err) + assert.False(t, dropped) expected := mapstr.M{ "cloud": mapstr.M{ @@ -408,7 +411,8 @@ func TestRetrieveGCEMetadataInK8s(t *testing.T) { }, }, } - assert.Equal(t, expected, actual.Fields) + ed.Apply() + assert.Equal(t, expected, event.Fields) } func TestRetrieveGCEMetadataInK8sNotOverriden(t *testing.T) { @@ -429,21 +433,20 @@ func TestRetrieveGCEMetadataInK8sNotOverriden(t *testing.T) { t.Fatal(err) } - actual, err := p.Run( - &beat.Event{ - Fields: mapstr.M{ - "orchestrator": mapstr.M{ - "cluster": mapstr.M{ - "name": "production-marketing-k8s", - "url": "https://35.223.150.35", - }, + event := &beat.Event{ + Fields: mapstr.M{ + "orchestrator": mapstr.M{ + "cluster": mapstr.M{ + "name": "production-marketing-k8s", + "url": "https://35.223.150.35", }, }, }, - ) - if err != nil { - t.Fatal(err) } + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) + assert.NoError(t, err) + assert.False(t, dropped) expected := mapstr.M{ "cloud": mapstr.M{ @@ -474,7 +477,8 @@ func TestRetrieveGCEMetadataInK8sNotOverriden(t *testing.T) { }, }, } - assert.Equal(t, expected, actual.Fields) + ed.Apply() + assert.Equal(t, expected, event.Fields) } func TestRetrieveGCEMetadataInK8sPartial(t *testing.T) { @@ -495,10 +499,11 @@ func TestRetrieveGCEMetadataInK8sPartial(t *testing.T) { t.Fatal(err) } - actual, err := p.Run(&beat.Event{Fields: mapstr.M{}}) - if err != nil { - t.Fatal(err) - } + event := &beat.Event{Fields: mapstr.M{}} + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) + assert.NoError(t, err) + assert.False(t, dropped) expected := mapstr.M{ "cloud": mapstr.M{ @@ -528,5 +533,6 @@ func TestRetrieveGCEMetadataInK8sPartial(t *testing.T) { }, }, } - assert.Equal(t, expected, actual.Fields) + ed.Apply() + assert.Equal(t, expected, event.Fields) } diff --git a/libbeat/processors/add_cloud_metadata/provider_hetzner_cloud_test.go b/libbeat/processors/add_cloud_metadata/provider_hetzner_cloud_test.go index f62f6629c2c9..387f56ff709c 100644 --- a/libbeat/processors/add_cloud_metadata/provider_hetzner_cloud_test.go +++ b/libbeat/processors/add_cloud_metadata/provider_hetzner_cloud_test.go @@ -76,10 +76,11 @@ func assertHetzner(t *testing.T, config *conf.C) { t.Fatal(err) } - actual, err := p.Run(&beat.Event{Fields: mapstr.M{}}) - if err != nil { - t.Fatal(err) - } + event := &beat.Event{Fields: mapstr.M{}} + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) + assert.NoError(t, err) + assert.False(t, dropped) expected := mapstr.M{ "cloud": mapstr.M{ @@ -95,5 +96,6 @@ func assertHetzner(t *testing.T, config *conf.C) { }, }, } - assert.Equal(t, expected, actual.Fields) + ed.Apply() + assert.Equal(t, expected, event.Fields) } diff --git a/libbeat/processors/add_cloud_metadata/provider_huawei_cloud_test.go b/libbeat/processors/add_cloud_metadata/provider_huawei_cloud_test.go index 0ae6fc332f09..9de85ac1aa6f 100644 --- a/libbeat/processors/add_cloud_metadata/provider_huawei_cloud_test.go +++ b/libbeat/processors/add_cloud_metadata/provider_huawei_cloud_test.go @@ -76,10 +76,11 @@ func TestRetrieveHuaweiCloudMetadata(t *testing.T) { t.Fatal(err) } - actual, err := p.Run(&beat.Event{Fields: mapstr.M{}}) - if err != nil { - t.Fatal(err) - } + event := &beat.Event{Fields: mapstr.M{}} + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) + assert.NoError(t, err) + assert.False(t, dropped) expected := mapstr.M{ "cloud": mapstr.M{ @@ -94,5 +95,6 @@ func TestRetrieveHuaweiCloudMetadata(t *testing.T) { }, }, } - assert.Equal(t, expected, actual.Fields) + ed.Apply() + assert.Equal(t, expected, event.Fields) } diff --git a/libbeat/processors/add_cloud_metadata/provider_openstack_nova_test.go b/libbeat/processors/add_cloud_metadata/provider_openstack_nova_test.go index 09c52d150665..3df1d5b38e8b 100644 --- a/libbeat/processors/add_cloud_metadata/provider_openstack_nova_test.go +++ b/libbeat/processors/add_cloud_metadata/provider_openstack_nova_test.go @@ -94,10 +94,11 @@ func assertOpenstackNova(t *testing.T, config *conf.C) { t.Fatal(err) } - actual, err := p.Run(&beat.Event{Fields: mapstr.M{}}) - if err != nil { - t.Fatal(err) - } + event := &beat.Event{Fields: mapstr.M{}} + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) + assert.NoError(t, err) + assert.False(t, dropped) expected := mapstr.M{ "cloud": mapstr.M{ @@ -115,5 +116,6 @@ func assertOpenstackNova(t *testing.T, config *conf.C) { }, }, } - assert.Equal(t, expected, actual.Fields) + ed.Apply() + assert.Equal(t, expected, event.Fields) } diff --git a/libbeat/processors/add_cloud_metadata/provider_tencent_cloud_test.go b/libbeat/processors/add_cloud_metadata/provider_tencent_cloud_test.go index 304fc492e20f..d70a743ccf4c 100644 --- a/libbeat/processors/add_cloud_metadata/provider_tencent_cloud_test.go +++ b/libbeat/processors/add_cloud_metadata/provider_tencent_cloud_test.go @@ -69,10 +69,11 @@ func TestRetrieveQCloudMetadata(t *testing.T) { t.Fatal(err) } - actual, err := p.Run(&beat.Event{Fields: mapstr.M{}}) - if err != nil { - t.Fatal(err) - } + event := &beat.Event{Fields: mapstr.M{}} + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) + assert.NoError(t, err) + assert.False(t, dropped) expected := mapstr.M{ "cloud": mapstr.M{ @@ -87,5 +88,6 @@ func TestRetrieveQCloudMetadata(t *testing.T) { }, }, } - assert.Equal(t, expected, actual.Fields) + ed.Apply() + assert.Equal(t, expected, event.Fields) } diff --git a/libbeat/processors/add_docker_metadata/add_docker_metadata.go b/libbeat/processors/add_docker_metadata/add_docker_metadata.go index d670713894da..9e3c48afb598 100644 --- a/libbeat/processors/add_docker_metadata/add_docker_metadata.go +++ b/libbeat/processors/add_docker_metadata/add_docker_metadata.go @@ -133,21 +133,21 @@ func lazyCgroupCacheInit(d *addDockerMetadata) { } } -func (d *addDockerMetadata) Run(event *beat.Event) (*beat.Event, error) { +func (d *addDockerMetadata) Run(event *beat.EventEditor) (dropped bool, err error) { if !d.dockerAvailable { - return event, nil + return false, nil } var cid string - var err error // Extract CID from the filepath contained in the "log.file.path" field. if d.sourceProcessor != nil { - lfp, _ := event.Fields.GetValue("log.file.path") + lfp, _ := event.GetValue("log.file.path") if lfp != nil { - event, err = d.sourceProcessor.Run(event) + // should never drop events + _, err = d.sourceProcessor.Run(event) if err != nil { d.log.Debugf("Error while extracting container ID from source path: %v", err) - return event, nil + return false, nil } if v, err := event.GetValue(dockerContainerIDKey); err == nil { @@ -160,7 +160,7 @@ func (d *addDockerMetadata) Run(event *beat.Event) (*beat.Event, error) { if cid == "" && len(d.pidFields) > 0 { id, err := d.lookupContainerIDByPID(event) if err != nil { - return nil, fmt.Errorf("error reading container ID: %w", err) + return true, fmt.Errorf("error reading container ID: %w", err) } if id != "" { cid = id @@ -184,7 +184,7 @@ func (d *addDockerMetadata) Run(event *beat.Event) (*beat.Event, error) { } if cid == "" { - return event, nil + return false, nil } container := d.watcher.Container(cid) @@ -207,12 +207,13 @@ func (d *addDockerMetadata) Run(event *beat.Event) (*beat.Event, error) { _, _ = meta.Put("container.id", container.ID) _, _ = meta.Put("container.image.name", container.Image) _, _ = meta.Put("container.name", container.Name) - event.Fields.DeepUpdate(meta.Clone()) + + event.DeepUpdate(meta.Clone()) } else { d.log.Debugf("Container not found: cid=%s", cid) } - return event, nil + return false, nil } func (d *addDockerMetadata) Close() error { @@ -237,7 +238,7 @@ func (d *addDockerMetadata) String() string { // lookupContainerIDByPID finds the container ID based on PID fields contained // in the event. -func (d *addDockerMetadata) lookupContainerIDByPID(event *beat.Event) (string, error) { +func (d *addDockerMetadata) lookupContainerIDByPID(event *beat.EventEditor) (string, error) { var cgroups cgroup.PathList for _, field := range d.pidFields { v, err := event.GetValue(field) diff --git a/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go b/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go index 2b6663f71dc1..e273e7002449 100644 --- a/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go +++ b/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go @@ -71,10 +71,13 @@ func TestInitializationNoDocker(t *testing.T) { assert.NoError(t, err, "initializing add_docker_metadata processor") input := mapstr.M{} - result, err := p.Run(&beat.Event{Fields: input}) + event := &beat.Event{Fields: input} + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) assert.NoError(t, err, "processing an event") - - assert.Equal(t, mapstr.M{}, result.Fields) + assert.False(t, dropped) + ed.Apply() + assert.Equal(t, mapstr.M{}, event.Fields) } func TestInitialization(t *testing.T) { @@ -84,10 +87,13 @@ func TestInitialization(t *testing.T) { assert.NoError(t, err, "initializing add_docker_metadata processor") input := mapstr.M{} - result, err := p.Run(&beat.Event{Fields: input}) + event := &beat.Event{Fields: input} + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) assert.NoError(t, err, "processing an event") - - assert.Equal(t, mapstr.M{}, result.Fields) + assert.False(t, dropped) + ed.Apply() + assert.Equal(t, mapstr.M{}, event.Fields) } func TestNoMatch(t *testing.T) { @@ -102,10 +108,13 @@ func TestNoMatch(t *testing.T) { input := mapstr.M{ "field": "value", } - result, err := p.Run(&beat.Event{Fields: input}) + event := &beat.Event{Fields: input} + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) assert.NoError(t, err, "processing an event") - - assert.Equal(t, mapstr.M{"field": "value"}, result.Fields) + assert.False(t, dropped) + ed.Apply() + assert.Equal(t, mapstr.M{"field": "value"}, event.Fields) } func TestMatchNoContainer(t *testing.T) { @@ -120,10 +129,13 @@ func TestMatchNoContainer(t *testing.T) { input := mapstr.M{ "foo": "garbage", } - result, err := p.Run(&beat.Event{Fields: input}) + event := &beat.Event{Fields: input} + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) assert.NoError(t, err, "processing an event") - - assert.Equal(t, mapstr.M{"foo": "garbage"}, result.Fields) + assert.False(t, dropped) + ed.Apply() + assert.Equal(t, mapstr.M{"foo": "garbage"}, event.Fields) } func TestMatchContainer(t *testing.T) { @@ -151,9 +163,12 @@ func TestMatchContainer(t *testing.T) { input := mapstr.M{ "foo": "container_id", } - result, err := p.Run(&beat.Event{Fields: input}) + event := &beat.Event{Fields: input} + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) assert.NoError(t, err, "processing an event") - + assert.False(t, dropped) + ed.Apply() assert.EqualValues(t, mapstr.M{ "container": mapstr.M{ "id": "container_id", @@ -172,7 +187,7 @@ func TestMatchContainer(t *testing.T) { "name": "name", }, "foo": "container_id", - }, result.Fields) + }, event.Fields) } func TestMatchContainerWithDedot(t *testing.T) { @@ -199,9 +214,12 @@ func TestMatchContainerWithDedot(t *testing.T) { input := mapstr.M{ "foo": "container_id", } - result, err := p.Run(&beat.Event{Fields: input}) + event := &beat.Event{Fields: input} + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) assert.NoError(t, err, "processing an event") - + assert.False(t, dropped) + ed.Apply() assert.EqualValues(t, mapstr.M{ "container": mapstr.M{ "id": "container_id", @@ -216,7 +234,7 @@ func TestMatchContainerWithDedot(t *testing.T) { "name": "name", }, "foo": "container_id", - }, result.Fields) + }, event.Fields) } func TestMatchSource(t *testing.T) { @@ -253,9 +271,12 @@ func TestMatchSource(t *testing.T) { }, } - result, err := p.Run(&beat.Event{Fields: input}) + event := &beat.Event{Fields: input} + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) assert.NoError(t, err, "processing an event") - + assert.False(t, dropped) + ed.Apply() assert.EqualValues(t, mapstr.M{ "container": mapstr.M{ "id": "8c147fdfab5a2608fe513d10294bf77cb502a231da9725093a155bd25cd1f14b", @@ -273,7 +294,7 @@ func TestMatchSource(t *testing.T) { "path": inputSource, }, }, - }, result.Fields) + }, event.Fields) } func TestDisableSource(t *testing.T) { @@ -300,11 +321,14 @@ func TestDisableSource(t *testing.T) { input := mapstr.M{ "source": "/var/lib/docker/containers/8c147fdfab5a2608fe513d10294bf77cb502a231da9725093a155bd25cd1f14b/foo.log", } - result, err := p.Run(&beat.Event{Fields: input}) + event := &beat.Event{Fields: input} + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) assert.NoError(t, err, "processing an event") - + assert.False(t, dropped) + ed.Apply() // remains unchanged - assert.EqualValues(t, input, result.Fields) + assert.EqualValues(t, input, event.Fields) } func TestMatchPIDs(t *testing.T) { @@ -345,9 +369,13 @@ func TestMatchPIDs(t *testing.T) { expected := mapstr.M{} expected.DeepUpdate(input) - result, err := p.Run(&beat.Event{Fields: input}) + event := &beat.Event{Fields: input} + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) assert.NoError(t, err, "processing an event") - assert.EqualValues(t, expected, result.Fields) + assert.False(t, dropped) + ed.Apply() + assert.EqualValues(t, expected, event.Fields) }) t.Run("pid does not exist", func(t *testing.T) { @@ -357,9 +385,13 @@ func TestMatchPIDs(t *testing.T) { expected := mapstr.M{} expected.DeepUpdate(input) - result, err := p.Run(&beat.Event{Fields: input}) + event := &beat.Event{Fields: input} + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) assert.NoError(t, err, "processing an event") - assert.EqualValues(t, expected, result.Fields) + assert.False(t, dropped) + ed.Apply() + assert.EqualValues(t, expected, event.Fields) }) t.Run("pid is containerized", func(t *testing.T) { @@ -370,9 +402,13 @@ func TestMatchPIDs(t *testing.T) { expected.DeepUpdate(dockerMetadata) expected.DeepUpdate(fields) - result, err := p.Run(&beat.Event{Fields: fields}) + event := &beat.Event{Fields: fields} + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) assert.NoError(t, err, "processing an event") - assert.EqualValues(t, expected, result.Fields) + assert.False(t, dropped) + ed.Apply() + assert.EqualValues(t, expected, event.Fields) }) t.Run("pid exited and ppid is containerized", func(t *testing.T) { @@ -384,9 +420,13 @@ func TestMatchPIDs(t *testing.T) { expected.DeepUpdate(dockerMetadata) expected.DeepUpdate(fields) - result, err := p.Run(&beat.Event{Fields: fields}) + event := &beat.Event{Fields: fields} + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) assert.NoError(t, err, "processing an event") - assert.EqualValues(t, expected, result.Fields) + assert.False(t, dropped) + ed.Apply() + assert.EqualValues(t, expected, event.Fields) }) t.Run("cgroup error", func(t *testing.T) { @@ -396,9 +436,13 @@ func TestMatchPIDs(t *testing.T) { expected := mapstr.M{} expected.DeepUpdate(fields) - result, err := p.Run(&beat.Event{Fields: fields}) + event := &beat.Event{Fields: fields} + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) assert.NoError(t, err, "processing an event") - assert.EqualValues(t, expected, result.Fields) + assert.False(t, dropped) + ed.Apply() + assert.EqualValues(t, expected, event.Fields) }) } diff --git a/libbeat/processors/add_host_metadata/add_host_metadata.go b/libbeat/processors/add_host_metadata/add_host_metadata.go index db3cbbc5ee30..7bea25fec559 100644 --- a/libbeat/processors/add_host_metadata/add_host_metadata.go +++ b/libbeat/processors/add_host_metadata/add_host_metadata.go @@ -121,23 +121,23 @@ func New(cfg *config.C) (beat.Processor, error) { } // Run enriches the given event with the host metadata -func (p *addHostMetadata) Run(event *beat.Event) (*beat.Event, error) { +func (p *addHostMetadata) Run(event *beat.EventEditor) (dropped bool, err error) { // check replace_host_fields field if !p.config.ReplaceFields && skipAddingHostMetadata(event) { - return event, nil + return false, nil } - err := p.loadData(true, features.FQDN()) + err = p.loadData(true, features.FQDN()) if err != nil { - return nil, fmt.Errorf("error loading data during event update: %w", err) + return true, fmt.Errorf("error loading data during event update: %w", err) } - event.Fields.DeepUpdate(p.data.Get().Clone()) + event.DeepUpdate(p.data.Get().Clone()) if len(p.geoData) > 0 { - event.Fields.DeepUpdate(p.geoData) + event.DeepUpdate(p.geoData) } - return event, nil + return false, nil } // Ideally we'd be able to implement the Closer interface here and @@ -283,9 +283,9 @@ func (p *addHostMetadata) updateOrExpire(useFQDN bool) { } -func skipAddingHostMetadata(event *beat.Event) bool { +func skipAddingHostMetadata(event *beat.EventEditor) bool { // If host fields exist(besides host.name added by libbeat) in event, skip add_host_metadata. - hostFields, err := event.Fields.GetValue("host") + hostFields, err := event.GetValue("host") // Don't skip if there are no fields if err != nil || hostFields == nil { diff --git a/libbeat/processors/add_host_metadata/add_host_metadata_test.go b/libbeat/processors/add_host_metadata/add_host_metadata_test.go index c90feb771851..50868030b258 100644 --- a/libbeat/processors/add_host_metadata/add_host_metadata_test.go +++ b/libbeat/processors/add_host_metadata/add_host_metadata_test.go @@ -60,30 +60,32 @@ func TestConfigDefault(t *testing.T) { return } - newEvent, err := p.Run(event) + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) assert.NoError(t, err) - - v, err := newEvent.GetValue("host.os.family") + assert.False(t, dropped) + ed.Apply() + v, err := event.GetValue("host.os.family") assert.NoError(t, err) assert.NotNil(t, v) - v, err = newEvent.GetValue("host.os.kernel") + v, err = event.GetValue("host.os.kernel") assert.NoError(t, err) assert.NotNil(t, v) - v, err = newEvent.GetValue("host.os.name") + v, err = event.GetValue("host.os.name") assert.NoError(t, err) assert.NotNil(t, v) - v, err = newEvent.GetValue("host.ip") + v, err = event.GetValue("host.ip") assert.NoError(t, err) assert.NotNil(t, v) - v, err = newEvent.GetValue("host.mac") + v, err = event.GetValue("host.mac") assert.NoError(t, err) assert.NotNil(t, v) - v, err = newEvent.GetValue("host.os.type") + v, err = event.GetValue("host.os.type") assert.NoError(t, err) assert.NotNil(t, v) } @@ -107,30 +109,33 @@ func TestConfigNetInfoDisabled(t *testing.T) { return } - newEvent, err := p.Run(event) + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) assert.NoError(t, err) + assert.False(t, dropped) + ed.Apply() - v, err := newEvent.GetValue("host.os.family") + v, err := event.GetValue("host.os.family") assert.NoError(t, err) assert.NotNil(t, v) - v, err = newEvent.GetValue("host.os.kernel") + v, err = event.GetValue("host.os.kernel") assert.NoError(t, err) assert.NotNil(t, v) - v, err = newEvent.GetValue("host.os.name") + v, err = event.GetValue("host.os.name") assert.NoError(t, err) assert.NotNil(t, v) - v, err = newEvent.GetValue("host.ip") + v, err = event.GetValue("host.ip") assert.Error(t, err) assert.Nil(t, v) - v, err = newEvent.GetValue("host.mac") + v, err = event.GetValue("host.mac") assert.Error(t, err) assert.Nil(t, v) - v, err = newEvent.GetValue("host.os.type") + v, err = event.GetValue("host.os.type") assert.NoError(t, err) assert.NotNil(t, v) } @@ -151,14 +156,17 @@ func TestConfigName(t *testing.T) { p, err := New(testConfig) require.NoError(t, err) - newEvent, err := p.Run(event) + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) assert.NoError(t, err) + assert.False(t, dropped) + ed.Apply() for configKey, configValue := range config { t.Run(fmt.Sprintf("Check of %s", configKey), func(t *testing.T) { - v, err := newEvent.GetValue(fmt.Sprintf("host.%s", configKey)) + v, err := event.GetValue(fmt.Sprintf("host.%s", configKey)) assert.NoError(t, err) - assert.Equal(t, configValue, v, "Could not find in %s", newEvent) + assert.Equal(t, configValue, v, "Could not find in %s", event) }) } } @@ -186,10 +194,13 @@ func TestConfigGeoEnabled(t *testing.T) { p, err := New(testConfig) require.NoError(t, err) - newEvent, err := p.Run(event) + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) assert.NoError(t, err) + assert.False(t, dropped) + ed.Apply() - eventGeoField, err := newEvent.GetValue("host.geo") + eventGeoField, err := event.GetValue("host.geo") require.NoError(t, err) assert.Len(t, eventGeoField, len(config)) @@ -209,11 +220,13 @@ func TestConfigGeoDisabled(t *testing.T) { p, err := New(testConfig) require.NoError(t, err) - newEvent, err := p.Run(event) - - require.NoError(t, err) + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) + assert.NoError(t, err) + assert.False(t, dropped) + ed.Apply() - eventGeoField, err := newEvent.GetValue("host.geo") + eventGeoField, err := event.GetValue("host.geo") assert.Error(t, err) assert.Equal(t, nil, eventGeoField) } @@ -235,14 +248,14 @@ func TestEventWithReplaceFieldsFalse(t *testing.T) { cases := []struct { title string - event beat.Event + event *beat.Event hostLengthLargerThanOne bool hostLengthEqualsToOne bool expectedHostFieldLength int }{ { "replace_fields=false with only host.name", - beat.Event{ + &beat.Event{ Fields: mapstr.M{ "host": mapstr.M{ "name": hostName, @@ -255,7 +268,7 @@ func TestEventWithReplaceFieldsFalse(t *testing.T) { }, { "replace_fields=false with only host.id", - beat.Event{ + &beat.Event{ Fields: mapstr.M{ "host": mapstr.M{ "id": hostID, @@ -268,7 +281,7 @@ func TestEventWithReplaceFieldsFalse(t *testing.T) { }, { "replace_fields=false with host.name and host.id", - beat.Event{ + &beat.Event{ Fields: mapstr.M{ "host": mapstr.M{ "name": hostName, @@ -284,10 +297,13 @@ func TestEventWithReplaceFieldsFalse(t *testing.T) { for _, c := range cases { t.Run(c.title, func(t *testing.T) { - newEvent, err := p.Run(&c.event) + ed := beat.NewEventEditor(c.event) + dropped, err := p.Run(ed) assert.NoError(t, err) + assert.False(t, dropped) + ed.Apply() - v, err := newEvent.GetValue("host") + v, err := c.event.GetValue("host") assert.NoError(t, err) assert.Equal(t, c.hostLengthLargerThanOne, len(v.(mapstr.M)) > 1) assert.Equal(t, c.hostLengthEqualsToOne, len(v.(mapstr.M)) == 1) @@ -315,13 +331,13 @@ func TestEventWithReplaceFieldsTrue(t *testing.T) { cases := []struct { title string - event beat.Event + event *beat.Event hostLengthLargerThanOne bool hostLengthEqualsToOne bool }{ { "replace_fields=true with host.name", - beat.Event{ + &beat.Event{ Fields: mapstr.M{ "host": mapstr.M{ "name": hostName, @@ -333,7 +349,7 @@ func TestEventWithReplaceFieldsTrue(t *testing.T) { }, { "replace_fields=true with host.id", - beat.Event{ + &beat.Event{ Fields: mapstr.M{ "host": mapstr.M{ "id": hostID, @@ -345,7 +361,7 @@ func TestEventWithReplaceFieldsTrue(t *testing.T) { }, { "replace_fields=true with host.name and host.id", - beat.Event{ + &beat.Event{ Fields: mapstr.M{ "host": mapstr.M{ "name": hostName, @@ -360,10 +376,13 @@ func TestEventWithReplaceFieldsTrue(t *testing.T) { for _, c := range cases { t.Run(c.title, func(t *testing.T) { - newEvent, err := p.Run(&c.event) + ed := beat.NewEventEditor(c.event) + dropped, err := p.Run(ed) assert.NoError(t, err) + assert.False(t, dropped) + ed.Apply() - v, err := newEvent.GetValue("host") + v, err := c.event.GetValue("host") assert.NoError(t, err) assert.Equal(t, c.hostLengthLargerThanOne, len(v.(mapstr.M)) > 1) assert.Equal(t, c.hostLengthEqualsToOne, len(v.(mapstr.M)) == 1) @@ -384,12 +403,12 @@ func TestSkipAddingHostMetadata(t *testing.T) { cases := []struct { title string - event beat.Event + event *beat.Event expectedSkip bool }{ { "event only with host.name", - beat.Event{ + &beat.Event{ Fields: mapstr.M{ "host": mapstr.M{ "name": hostName, @@ -400,7 +419,7 @@ func TestSkipAddingHostMetadata(t *testing.T) { }, { "event only with host.id", - beat.Event{ + &beat.Event{ Fields: mapstr.M{ "host": mapstr.M{ "id": hostID, @@ -411,7 +430,7 @@ func TestSkipAddingHostMetadata(t *testing.T) { }, { "event with host.name and host.id", - beat.Event{ + &beat.Event{ Fields: mapstr.M{ "host": mapstr.M{ "name": hostName, @@ -423,14 +442,14 @@ func TestSkipAddingHostMetadata(t *testing.T) { }, { "event without host field", - beat.Event{ + &beat.Event{ Fields: mapstr.M{}, }, false, }, { "event with field type map[string]string hostID", - beat.Event{ + &beat.Event{ Fields: mapstr.M{ "host": hostIDMap, }, @@ -439,7 +458,7 @@ func TestSkipAddingHostMetadata(t *testing.T) { }, { "event with field type map[string]string host name", - beat.Event{ + &beat.Event{ Fields: mapstr.M{ "host": hostNameMap, }, @@ -448,7 +467,7 @@ func TestSkipAddingHostMetadata(t *testing.T) { }, { "event with field type map[string]string host ID and name", - beat.Event{ + &beat.Event{ Fields: mapstr.M{ "host": hostIDNameMap, }, @@ -457,7 +476,7 @@ func TestSkipAddingHostMetadata(t *testing.T) { }, { "event with field type string", - beat.Event{ + &beat.Event{ Fields: mapstr.M{ "host": "string", }, @@ -468,7 +487,8 @@ func TestSkipAddingHostMetadata(t *testing.T) { for _, c := range cases { t.Run(c.title, func(t *testing.T) { - skip := skipAddingHostMetadata(&c.event) + ed := beat.NewEventEditor(c.event) + skip := skipAddingHostMetadata(ed) assert.Equal(t, c.expectedSkip, skip) }) } @@ -516,11 +536,15 @@ func TestFQDNEventSync(t *testing.T) { for i := 0; i < 10; i++ { checkWait.Add(1) go func() { - resp, err := p.Run(&beat.Event{ + event := &beat.Event{ Fields: mapstr.M{}, - }) + } + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) require.NoError(t, err) - name, err := resp.Fields.GetValue("host.name") + require.False(t, dropped) + ed.Apply() + name, err := event.GetValue("host.name") require.NoError(t, err) require.Equal(t, "foo.bar.baz", name) checkWait.Done() @@ -594,10 +618,12 @@ func TestFQDNLookup(t *testing.T) { Fields: mapstr.M{}, Timestamp: time.Now(), } - newEvent, err := p.Run(event) + ed := beat.NewEventEditor(event) + dropped, err := p.Run(ed) require.NoError(t, err) - - v, err := newEvent.GetValue("host.name") + require.False(t, dropped) + ed.Apply() + v, err := event.GetValue("host.name") require.NoError(t, err) require.Equal(t, test.expectedHostName, v) }) diff --git a/libbeat/processors/add_id/add_id.go b/libbeat/processors/add_id/add_id.go index 9374a93e1661..062891a5016c 100644 --- a/libbeat/processors/add_id/add_id.go +++ b/libbeat/processors/add_id/add_id.go @@ -60,14 +60,14 @@ func New(cfg *conf.C) (beat.Processor, error) { } // Run enriches the given event with an ID -func (p *addID) Run(event *beat.Event) (*beat.Event, error) { +func (p *addID) Run(event *beat.EventEditor) (dropped bool, err error) { id := p.gen.NextID() if _, err := event.PutValue(p.config.TargetField, id); err != nil { - return nil, makeErrComputeID(err) + return true, makeErrComputeID(err) } - return event, nil + return false, nil } func (p *addID) String() string { diff --git a/libbeat/processors/add_id/add_id_test.go b/libbeat/processors/add_id/add_id_test.go index 18effb85205b..3891035f1444 100644 --- a/libbeat/processors/add_id/add_id_test.go +++ b/libbeat/processors/add_id/add_id_test.go @@ -33,11 +33,13 @@ func TestDefaultTargetField(t *testing.T) { assert.NoError(t, err) testEvent := &beat.Event{} - - newEvent, err := p.Run(testEvent) + ed := beat.NewEventEditor(testEvent) + dropped, err := p.Run(ed) assert.NoError(t, err) + assert.False(t, dropped) + ed.Apply() - v, err := newEvent.GetValue("@metadata._id") + v, err := testEvent.GetValue("@metadata._id") assert.NoError(t, err) assert.NotEmpty(t, v) } @@ -49,19 +51,20 @@ func TestNonDefaultTargetField(t *testing.T) { p, err := New(cfg) assert.NoError(t, err) - testEvent := &beat.Event{ - Fields: mapstr.M{}, - } - - newEvent, err := p.Run(testEvent) + testEvent := &beat.Event{} + ed := beat.NewEventEditor(testEvent) + dropped, err := p.Run(ed) assert.NoError(t, err) + assert.False(t, dropped) + ed.Apply() - v, err := newEvent.GetValue("foo") + v, err := testEvent.GetValue("foo") assert.NoError(t, err) assert.NotEmpty(t, v) - v, err = newEvent.GetValue("@metadata._id") - assert.NoError(t, err) + v, err = testEvent.GetValue("@metadata._id") + assert.Error(t, err) + assert.ErrorIs(t, err, mapstr.ErrKeyNotFound) assert.Empty(t, v) } @@ -72,17 +75,18 @@ func TestNonDefaultMetadataTarget(t *testing.T) { p, err := New(cfg) assert.NoError(t, err) - testEvent := &beat.Event{ - Meta: mapstr.M{}, - } - - newEvent, err := p.Run(testEvent) + testEvent := &beat.Event{} + ed := beat.NewEventEditor(testEvent) + dropped, err := p.Run(ed) assert.NoError(t, err) + assert.False(t, dropped) + ed.Apply() - v, err := newEvent.Meta.GetValue("foo") + v, err := testEvent.GetValue("@metadata.foo") assert.NoError(t, err) assert.NotEmpty(t, v) - v, err = newEvent.GetValue("@metadata._id") + v, err = testEvent.GetValue("@metadata._id") assert.Error(t, err) + assert.ErrorIs(t, err, mapstr.ErrKeyNotFound) } diff --git a/libbeat/processors/conditionals.go b/libbeat/processors/conditionals.go index e5e65660c8ef..035ddb9fc151 100644 --- a/libbeat/processors/conditionals.go +++ b/libbeat/processors/conditionals.go @@ -77,9 +77,9 @@ func NewConditionRule( } // Run executes this WhenProcessor. -func (r *WhenProcessor) Run(event *beat.Event) (*beat.Event, error) { +func (r *WhenProcessor) Run(event *beat.EventEditor) (dropped bool, err error) { if !(r.condition).Check(event) { - return event, nil + return false, nil } return r.p.Run(event) } @@ -162,13 +162,13 @@ func NewIfElseThenProcessor(cfg *config.C) (*IfThenElseProcessor, error) { // Run checks the if condition and executes the processors attached to the // then statement or the else statement based on the condition. -func (p *IfThenElseProcessor) Run(event *beat.Event) (*beat.Event, error) { +func (p *IfThenElseProcessor) Run(event *beat.EventEditor) (dropped bool, err error) { if p.cond.Check(event) { return p.then.Run(event) } else if p.els != nil { return p.els.Run(event) } - return event, nil + return false, nil } func (p *IfThenElseProcessor) String() string { diff --git a/libbeat/processors/conditionals_test.go b/libbeat/processors/conditionals_test.go index 11544625a2e3..d0a9dd2eb858 100644 --- a/libbeat/processors/conditionals_test.go +++ b/libbeat/processors/conditionals_test.go @@ -33,9 +33,9 @@ type countFilter struct { N int } -func (c *countFilter) Run(e *beat.Event) (*beat.Event, error) { +func (c *countFilter) Run(e *beat.EventEditor) (dropped bool, err error) { c.N++ - return e, nil + return false, nil } func (c *countFilter) String() string { return "count" } @@ -100,10 +100,10 @@ func TestWhenProcessor(t *testing.T) { } for _, fields := range test.events { - event := &beat.Event{ + event := beat.NewEventEditor(&beat.Event{ Timestamp: time.Now(), Fields: fields, - } + }) _, err := filter.Run(event) if err != nil { t.Error(err) @@ -149,11 +149,13 @@ func testProcessors(t *testing.T, cases map[string]testCase) { t.Fatal(err) } - result, err := processor.Run(&beat.Event{Fields: test.event.Clone()}) + ed := beat.NewEventEditor(&beat.Event{Fields: test.event}) + dropped, err := processor.Run(ed) if err != nil { t.Fatal(err) } - assert.Equal(t, test.want, result.Fields) + assert.False(t, dropped) + assert.Equal(t, test.want, ed.Fields()) }) } } diff --git a/libbeat/processors/namespace_test.go b/libbeat/processors/namespace_test.go index 07c62328c2c1..a420480c4133 100644 --- a/libbeat/processors/namespace_test.go +++ b/libbeat/processors/namespace_test.go @@ -29,7 +29,7 @@ import ( type testFilterRule struct { str func() string - run func(*beat.Event) (*beat.Event, error) + run func(*beat.EventEditor) (dropped bool, err error) } func TestNamespace(t *testing.T) { @@ -132,9 +132,9 @@ func (r *testFilterRule) String() string { return r.str() } -func (r *testFilterRule) Run(evt *beat.Event) (*beat.Event, error) { +func (r *testFilterRule) Run(evt *beat.EventEditor) (dropped bool, err error) { if r.run == nil { - return evt, nil + return false, nil } return r.Run(evt) } diff --git a/libbeat/processors/processor.go b/libbeat/processors/processor.go index d1f84aee33e3..dafe3c00a5f7 100644 --- a/libbeat/processors/processor.go +++ b/libbeat/processors/processor.go @@ -158,22 +158,31 @@ func (procs *Processors) Close() error { return errs.Err() } -// Run executes the all processors serially and returns the event and possibly -// an error. If the event has been dropped (canceled) by a processor in the -// list then a nil event is returned. -func (procs *Processors) Run(event *beat.Event) (*beat.Event, error) { - var err error +// Run executes all processors from the list serially and applies changes of each +// successfully run processor to the event through the event editor. +// If one of the processors drops the event, this processor also returns `dropped=true` and no error. +func (procs *Processors) Run(event *beat.EventEditor) (dropped bool, err error) { for _, p := range procs.List { - event, err = p.Run(event) - if err != nil { - return event, fmt.Errorf("failed applying processor %v: %w", p, err) + dropped, err = p.Run(event) + if dropped { + return dropped, err } - if event == nil { - // Drop. - return nil, nil + if err != nil { + // if the processor returns an error + // most-likely the changes are incomplete and we don't want + // them to be applied to the event. + event.Reset() + // however, we want to document the error if the processor returned an `EventError` + ee, isEventError := err.(beat.EventError) + if isEventError { + event.AddError(ee) + event.Apply() + } + return dropped, fmt.Errorf("failed applying processor %s: %w", p.String(), err) } + event.Apply() } - return event, nil + return false, nil } func (procs Processors) String() string { diff --git a/libbeat/processors/processor_test.go b/libbeat/processors/processor_test.go index b9370eb4489a..223f7a6b20c4 100644 --- a/libbeat/processors/processor_test.go +++ b/libbeat/processors/processor_test.go @@ -128,9 +128,9 @@ func TestIncludeFields(t *testing.T) { "type": "process", }, } - - processedEvent, err := processors.Run(event) - if err != nil { + ed := beat.NewEventEditor(event) + dropped, err := processors.Run(ed) + if err != nil || dropped { t.Fatal(err) } @@ -148,8 +148,8 @@ func TestIncludeFields(t *testing.T) { }, "type": "process", } - - assert.Equal(t, expectedEvent, processedEvent.Fields) + ed.Apply() + assert.Equal(t, expectedEvent, event.Fields) } func TestIncludeFields1(t *testing.T) { @@ -198,13 +198,14 @@ func TestIncludeFields1(t *testing.T) { }, } - processedEvent, _ := processors.Run(event) + ed := beat.NewEventEditor(event) + _, _ = processors.Run(ed) expectedEvent := mapstr.M{ "type": "process", } - - assert.Equal(t, expectedEvent, processedEvent.Fields) + ed.Apply() + assert.Equal(t, expectedEvent, event.Fields) } func TestDropFields(t *testing.T) { @@ -216,7 +217,7 @@ func TestDropFields(t *testing.T) { "beat.hostname": "mar", }, }, - "fields": []string{"proc.cpu.start_time", "mem", "proc.cmdline", "beat", "dd"}, + "fields": []string{"proc.cpu.start_time", "mem", "proc.cmdline", "beat"}, }, }, } @@ -251,7 +252,10 @@ func TestDropFields(t *testing.T) { }, } - processedEvent, _ := processors.Run(event) + ed := beat.NewEventEditor(event) + dropped, err := processors.Run(ed) + assert.NoError(t, err) + assert.False(t, dropped) expectedEvent := mapstr.M{ "proc": mapstr.M{ @@ -265,7 +269,7 @@ func TestDropFields(t *testing.T) { "type": "process", } - assert.Equal(t, expectedEvent, processedEvent.Fields) + assert.Equal(t, expectedEvent, event.Fields) } func TestMultipleIncludeFields(t *testing.T) { @@ -294,7 +298,6 @@ func TestMultipleIncludeFields(t *testing.T) { event1 := &beat.Event{ Timestamp: time.Now(), Fields: mapstr.M{ - "@timestamp": "2016-01-24T18:35:19.308Z", "beat": mapstr.M{ "hostname": "mar", "name": "my-shipper-1", @@ -358,11 +361,13 @@ func TestMultipleIncludeFields(t *testing.T) { "type": "process", } - actual1, _ := processors.Run(event1) - actual2, _ := processors.Run(event2) + ed1 := beat.NewEventEditor(event1) + ed2 := beat.NewEventEditor(event2) + _, _ = processors.Run(ed1) + _, _ = processors.Run(ed2) - assert.Equal(t, expected1, actual1.Fields) - assert.Equal(t, expected2, actual2.Fields) + assert.Equal(t, expected1, event1.Fields) + assert.Equal(t, expected2, event2.Fields) } func TestDropEvent(t *testing.T) { @@ -412,9 +417,10 @@ func TestDropEvent(t *testing.T) { }, } - processedEvent, _ := processors.Run(event) + ed := beat.NewEventEditor(event) + dropped, _ := processors.Run(ed) - assert.Nil(t, processedEvent) + assert.True(t, dropped) } func TestEmptyCondition(t *testing.T) { @@ -456,9 +462,10 @@ func TestEmptyCondition(t *testing.T) { }, } - processedEvent, _ := processors.Run(event) + ed := beat.NewEventEditor(event) + dropped, _ := processors.Run(ed) - assert.Nil(t, processedEvent) + assert.True(t, dropped) } func TestBadCondition(t *testing.T) { @@ -523,7 +530,8 @@ func TestDropMissingFields(t *testing.T) { yml := []map[string]interface{}{ { "drop_fields": map[string]interface{}{ - "fields": []string{"foo.bar", "proc.cpu", "proc.sss", "beat", "mem"}, + "ignore_missing": true, + "fields": []string{"foo.bar", "proc.cpu", "proc.sss", "beat", "mem"}, }, }, } @@ -558,7 +566,10 @@ func TestDropMissingFields(t *testing.T) { }, } - processedEvent, _ := processors.Run(event) + ed := beat.NewEventEditor(event) + dropped, err := processors.Run(ed) + assert.False(t, dropped) + assert.NoError(t, err) expectedEvent := mapstr.M{ "proc": mapstr.M{ @@ -567,7 +578,7 @@ func TestDropMissingFields(t *testing.T) { "type": "process", } - assert.Equal(t, expectedEvent, processedEvent.Fields) + assert.Equal(t, expectedEvent, event.Fields) } const ( @@ -578,10 +589,10 @@ const ( func BenchmarkProcessorsRun(b *testing.B) { processors := processors.NewList(nil) key1 := "added.field" - proc1 := actions.NewAddFields(mapstr.M{key1: "first"}, true, true) + proc1 := actions.NewAddFields(mapstr.M{key1: "first"}, true) processors.AddProcessor(proc1) key2 := "field-0.field-0" - proc2 := actions.NewAddFields(mapstr.M{key2: "second"}, true, true) + proc2 := actions.NewAddFields(mapstr.M{key2: "second"}, true) processors.AddProcessor(proc2) event := &beat.Event{ @@ -593,24 +604,26 @@ func BenchmarkProcessorsRun(b *testing.B) { generateFields(b, event.Meta, 100, 2) generateFields(b, event.Fields, 100, 2) + ed := beat.NewEventEditor(event) + var ( - processed *beat.Event - err error + dropped bool + err error ) b.Run("processors.Run", func(b *testing.B) { for i := 0; i < b.N; i++ { - processed, err = processors.Run(event) + dropped, err = processors.Run(ed) require.NoError(b, err) - require.NotNil(b, processed) + require.False(b, dropped) } }) - added, err := processed.GetValue(key1) + added, err := ed.GetValue(key1) require.NoError(b, err) require.Equal(b, "first", added) - added, err = processed.GetValue(key2) + added, err = ed.GetValue(key2) require.NoError(b, err) require.Equal(b, "second", added) } diff --git a/libbeat/processors/safe_processor.go b/libbeat/processors/safe_processor.go index a0bbf5824d07..2f9f9f0c2996 100644 --- a/libbeat/processors/safe_processor.go +++ b/libbeat/processors/safe_processor.go @@ -34,9 +34,9 @@ type SafeProcessor struct { } // Run allows to run processor only when `Close` was not called prior -func (p *SafeProcessor) Run(event *beat.Event) (*beat.Event, error) { +func (p *SafeProcessor) Run(event *beat.EventEditor) (dropped bool, err error) { if atomic.LoadUint32(&p.closed) == 1 { - return nil, ErrClosed + return true, ErrClosed } return p.Processor.Run(event) } diff --git a/libbeat/processors/safe_processor_test.go b/libbeat/processors/safe_processor_test.go index 98ad4e7dba86..adbe8aeb648b 100644 --- a/libbeat/processors/safe_processor_test.go +++ b/libbeat/processors/safe_processor_test.go @@ -32,9 +32,9 @@ type mockProcessor struct { runCount int } -func (p *mockProcessor) Run(event *beat.Event) (*beat.Event, error) { +func (p *mockProcessor) Run(event *beat.EventEditor) (bool, error) { p.runCount++ - return mockEvent, nil + return false, nil } func (p *mockProcessor) String() string { @@ -100,12 +100,13 @@ func TestSafeProcessor(t *testing.T) { t.Run("propagates Run to a processor", func(t *testing.T) { require.Equal(t, 0, p.runCount) - e, err := sp.Run(nil) + dropped, err := sp.Run(nil) require.NoError(t, err) - require.Equal(t, e, mockEvent) - e, err = sp.Run(nil) + require.False(t, dropped) + + dropped, err = sp.Run(nil) require.NoError(t, err) - require.Equal(t, e, mockEvent) + require.False(t, dropped) require.Equal(t, 2, p.runCount) }) @@ -123,8 +124,8 @@ func TestSafeProcessor(t *testing.T) { t.Run("does not propagate Run when closed", func(t *testing.T) { require.Equal(t, 2, p.runCount) // still 2 from the previous test case - e, err := sp.Run(nil) - require.Nil(t, e) + dropped, err := sp.Run(nil) + require.True(t, dropped) require.ErrorIs(t, err, ErrClosed) require.Equal(t, 2, p.runCount) }) diff --git a/libbeat/processors/script/javascript/beatevent_v0.go b/libbeat/processors/script/javascript/beatevent_v0.go index 82b011fb991e..b50d98be8f76 100644 --- a/libbeat/processors/script/javascript/beatevent_v0.go +++ b/libbeat/processors/script/javascript/beatevent_v0.go @@ -36,7 +36,7 @@ import ( type beatEventV0 struct { vm *goja.Runtime obj *goja.Object - inner *beat.Event + inner *beat.EventEditor cancelled bool } @@ -73,7 +73,7 @@ func newBeatEventV0Constructor(s Session) func(call goja.ConstructorCall) *goja. obj: call.This, } evt.init() - evt.reset(&beat.Event{Fields: fields}) + evt.reset(beat.NewEventEditor(&beat.Event{Fields: fields})) return nil } } @@ -89,16 +89,15 @@ func (e *beatEventV0) init() { } // reset the event so that it can be reused to wrap another event. -func (e *beatEventV0) reset(b *beat.Event) error { - e.inner = b +func (e *beatEventV0) reset(ed *beat.EventEditor) error { + e.inner = ed e.cancelled = false e.obj.Set("_private", e) - e.obj.Set("fields", e.vm.ToValue(e.inner.Fields)) return nil } -// Wrapped returns the wrapped beat.Event. -func (e *beatEventV0) Wrapped() *beat.Event { +// Wrapped returns the wrapped beat.EventEditor. +func (e *beatEventV0) Wrapped() *beat.EventEditor { return e.inner } @@ -116,8 +115,8 @@ func (e *beatEventV0) JSObject() goja.Value { func (e *beatEventV0) get(call goja.FunctionCall) goja.Value { a0 := call.Argument(0) if goja.IsUndefined(a0) { - // event.Get() is the same as event.fields (but slower). - return e.vm.ToValue(e.inner.Fields) + // event.Get() returns the whole fields map, it's slow and expensive + return e.vm.ToValue(e.inner.Fields()) } v, err := e.inner.GetValue(a0.String()) @@ -234,7 +233,7 @@ func (e *beatEventV0) tag(call goja.FunctionCall) goja.Value { tag := call.Argument(0).String() - if err := appendString(e.inner.Fields, "tags", tag, true); err != nil { + if err := appendString(e.inner, "tags", tag, true); err != nil { panic(err) } return goja.Undefined() @@ -255,24 +254,24 @@ func (e *beatEventV0) appendTo(call goja.FunctionCall) goja.Value { field := call.Argument(0).String() value := call.Argument(1).String() - if err := appendString(e.inner.Fields, field, value, false); err != nil { + if err := appendString(e.inner, field, value, false); err != nil { panic(err) } return goja.Undefined() } -func appendString(m mapstr.M, field, value string, alwaysArray bool) error { - list, _ := m.GetValue(field) +func appendString(ed *beat.EventEditor, field, value string, alwaysArray bool) error { + list, _ := ed.GetValue(field) switch v := list.(type) { case nil: if alwaysArray { - m.Put(field, []string{value}) + ed.PutValue(field, []string{value}) } else { - m.Put(field, value) + ed.PutValue(field, value) } case string: if value != v { - m.Put(field, []string{v, value}) + ed.PutValue(field, []string{v, value}) } case []string: for _, existingTag := range v { @@ -281,7 +280,7 @@ func appendString(m mapstr.M, field, value string, alwaysArray bool) error { return nil } } - m.Put(field, append(v, value)) + ed.PutValue(field, append(v, value)) case []interface{}: for _, existingTag := range v { if value == existingTag { @@ -289,7 +288,7 @@ func appendString(m mapstr.M, field, value string, alwaysArray bool) error { return nil } } - m.Put(field, append(v, value)) + ed.PutValue(field, append(v, value)) default: return fmt.Errorf("unexpected type %T found for %v field", list, field) } diff --git a/libbeat/processors/script/javascript/beatevent_v0_test.go b/libbeat/processors/script/javascript/beatevent_v0_test.go index 9d29a37b1b0f..fb26375cfdf9 100644 --- a/libbeat/processors/script/javascript/beatevent_v0_test.go +++ b/libbeat/processors/script/javascript/beatevent_v0_test.go @@ -26,7 +26,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/beats/v7/libbeat/beat/events" "github.com/elastic/beats/v7/libbeat/tests/resources" "github.com/elastic/elastic-agent-libs/mapstr" "github.com/elastic/elastic-agent-libs/monitoring" @@ -40,24 +39,17 @@ const ( type testCase struct { name string source string - assert func(t testing.TB, evt *beat.Event, err error) + assert func(t testing.TB, evt *beat.EventEditor, dropped bool, err error) } var eventV0Tests = []testCase{ { name: "Put", source: `evt.Put("hello", "world");`, - assert: func(t testing.TB, evt *beat.Event, err error) { - v, _ := evt.GetValue("hello") - assert.Equal(t, "world", v) - }, - }, - { - name: "Object Put Key", - source: `evt.fields["hello"] = "world";`, - assert: func(t testing.TB, evt *beat.Event, err error) { + assert: func(t testing.TB, evt *beat.EventEditor, dropped bool, err error) { v, _ := evt.GetValue("hello") assert.Equal(t, "world", v) + assert.False(t, dropped) }, }, { @@ -87,29 +79,22 @@ var eventV0Tests = []testCase{ throw "failed to get IP"; }`, }, - { - name: "fields get key", - source: ` - var ip = evt.fields.source.ip; - - if ("192.0.2.1" !== ip) { - throw "failed to get IP"; - }`, - }, { name: "Delete", source: `if (!evt.Delete("source.ip")) { throw "delete failed"; }`, - assert: func(t testing.TB, evt *beat.Event, err error) { + assert: func(t testing.TB, evt *beat.EventEditor, dropped bool, err error) { ip, _ := evt.GetValue("source.ip") assert.Nil(t, ip) + assert.False(t, dropped) }, }, { name: "Rename", source: `if (!evt.Rename("source", "destination")) { throw "rename failed"; }`, - assert: func(t testing.TB, evt *beat.Event, err error) { + assert: func(t testing.TB, evt *beat.EventEditor, dropped bool, err error) { ip, _ := evt.GetValue("destination.ip") assert.Equal(t, "192.0.2.1", ip) + assert.False(t, dropped) }, }, { @@ -121,48 +106,58 @@ var eventV0Tests = []testCase{ { name: "Put @metadata", source: `evt.Put("@metadata.foo", "bar");`, - assert: func(t testing.TB, evt *beat.Event, err error) { - assert.Equal(t, "bar", evt.Meta["foo"]) + assert: func(t testing.TB, evt *beat.EventEditor, dropped bool, err error) { + meta, _ := evt.GetValue("@metadata.foo") + assert.Equal(t, "bar", meta) + assert.False(t, dropped) }, }, { name: "Delete @metadata", source: `evt.Delete("@metadata.pipeline");`, - assert: func(t testing.TB, evt *beat.Event, err error) { - assert.Nil(t, evt.Meta[events.FieldMetaPipeline]) + assert: func(t testing.TB, evt *beat.EventEditor, dropped bool, err error) { + val, getErr := evt.GetValue("@metadata.pipeline") + assert.Nil(t, val) + assert.NoError(t, err) + assert.Error(t, getErr) + assert.ErrorIs(t, getErr, mapstr.ErrKeyNotFound) + assert.False(t, dropped) }, }, { name: "Cancel", source: `evt.Cancel();`, - assert: func(t testing.TB, evt *beat.Event, err error) { + assert: func(t testing.TB, evt *beat.EventEditor, dropped bool, err error) { assert.NoError(t, err) - assert.Nil(t, evt) + assert.True(t, dropped) }, }, { name: "Tag", source: `evt.Tag("foo"); evt.Tag("bar"); evt.Tag("foo");`, - assert: func(t testing.TB, evt *beat.Event, err error) { - if assert.NoError(t, err) { - assert.Equal(t, []string{"foo", "bar"}, evt.Fields["tags"]) - } + assert: func(t testing.TB, evt *beat.EventEditor, dropped bool, err error) { + assert.NoError(t, err) + val, getErr := evt.GetValue(mapstr.TagsKey) + assert.NoError(t, getErr) + assert.Equal(t, []string{"foo", "bar"}, val) + assert.False(t, dropped) }, }, { name: "AppendTo", source: `evt.AppendTo("source.ip", "10.0.0.1");`, - assert: func(t testing.TB, evt *beat.Event, err error) { + assert: func(t testing.TB, evt *beat.EventEditor, dropped bool, err error) { if assert.NoError(t, err) { srcIP, _ := evt.GetValue("source.ip") assert.Equal(t, []string{"192.0.2.1", "10.0.0.1"}, srcIP) } + assert.False(t, dropped) }, }, } -func testEvent() *beat.Event { - return &beat.Event{ +func testEvent() *beat.EventEditor { + return beat.NewEventEditor(&beat.Event{ Meta: mapstr.M{ "pipeline": "beat-1.2.3-module", }, @@ -171,7 +166,7 @@ func testEvent() *beat.Event { "ip": "192.0.2.1", }, }, - } + }) } func TestBeatEventV0(t *testing.T) { @@ -184,9 +179,10 @@ func TestBeatEventV0(t *testing.T) { t.Fatal(err) } - evt, err := p.Run(testEvent()) + evt := testEvent() + dropped, err := p.Run(evt) if tc.assert != nil { - tc.assert(t, evt, err) + tc.assert(t, evt, dropped, err) } else { assert.NoError(t, err) assert.NotNil(t, evt) diff --git a/libbeat/processors/script/javascript/javascript.go b/libbeat/processors/script/javascript/javascript.go index 527bd03e92ce..8937dc46be0c 100644 --- a/libbeat/processors/script/javascript/javascript.go +++ b/libbeat/processors/script/javascript/javascript.go @@ -164,31 +164,28 @@ func annotateError(id string, err error) error { // Run executes the processor on the given it event. It invokes the // process function defined in the Javascript source. -func (p *jsProcessor) Run(event *beat.Event) (*beat.Event, error) { +func (p *jsProcessor) Run(event *beat.EventEditor) (dropped bool, err error) { s := p.sessionPool.Get() defer p.sessionPool.Put(s) - var rtn *beat.Event - var err error - if p.stats == nil { - rtn, err = s.runProcessFunc(event) + dropped, err = s.runProcessFunc(event) } else { - rtn, err = p.runWithStats(s, event) + dropped, err = p.runWithStats(s, event) } - return rtn, annotateError(p.Tag, err) + return dropped, annotateError(p.Tag, err) } -func (p *jsProcessor) runWithStats(s *session, event *beat.Event) (*beat.Event, error) { +func (p *jsProcessor) runWithStats(s *session, event *beat.EventEditor) (dropped bool, err error) { start := time.Now() - event, err := s.runProcessFunc(event) + dropped, err = s.runProcessFunc(event) elapsed := time.Since(start) p.stats.processTime.Update(int64(elapsed)) if err != nil { p.stats.exceptions.Inc() } - return event, err + return dropped, err } func (p *jsProcessor) String() string { diff --git a/libbeat/processors/script/javascript/module/processor/chain.go b/libbeat/processors/script/javascript/module/processor/chain.go index 04982947b302..0057a9809551 100644 --- a/libbeat/processors/script/javascript/module/processor/chain.go +++ b/libbeat/processors/script/javascript/module/processor/chain.go @@ -169,11 +169,11 @@ func newNativeProcessor(constructor processors.Constructor, call gojaCall) (proc } func (p *nativeProcessor) run(event javascript.Event) error { - out, err := p.Processor.Run(event.Wrapped()) + dropped, err := p.Processor.Run(event.Wrapped()) if err != nil { return err } - if out == nil { + if dropped { event.Cancel() } return nil diff --git a/libbeat/processors/script/javascript/session.go b/libbeat/processors/script/javascript/session.go index 337d2eae7997..f7716838d938 100644 --- a/libbeat/processors/script/javascript/session.go +++ b/libbeat/processors/script/javascript/session.go @@ -60,14 +60,14 @@ type Event interface { // Wrapped returns the underlying beat.Event being wrapped. The wrapped // event is replaced each time a new event is processed. - Wrapped() *beat.Event + Wrapped() *beat.EventEditor // JSObject returns the Value that represents this object within the // runtime. JSObject() goja.Value // reset replaces the inner beat.Event and resets the state. - reset(*beat.Event) error + reset(*beat.EventEditor) error } // session is a javascript runtime environment used throughout the life of @@ -192,7 +192,7 @@ func (s *session) executeTestFunction() error { } // setEvent replaces the beat event handle present in the runtime. -func (s *session) setEvent(b *beat.Event) error { +func (s *session) setEvent(b *beat.EventEditor) error { if s.evt == nil { var err error s.evt, err = s.makeEvent(s) @@ -205,28 +205,28 @@ func (s *session) setEvent(b *beat.Event) error { } // runProcessFunc executes process() from the JS script. -func (s *session) runProcessFunc(b *beat.Event) (out *beat.Event, err error) { +func (s *session) runProcessFunc(b *beat.EventEditor) (dropped bool, err error) { defer func() { if r := recover(); r != nil { s.log.Errorw("The javascript processor caused an unexpected panic "+ "while processing an event. Recovering, but please report this.", - "event", mapstr.M{"original": b.Fields.String()}, + "event", b.String(), "panic", r, zap.Stack("stack")) if !s.evt.IsCancelled() { - out = b + b.Reset() } err = fmt.Errorf("unexpected panic in javascript processor: %v", r) if s.tagOnException != "" { - mapstr.AddTags(b.Fields, []string{s.tagOnException}) + b.AddTags(s.tagOnException) } - appendString(b.Fields, "error.message", err.Error(), false) + appendString(b, "error.message", err.Error(), false) } }() if err = s.setEvent(b); err != nil { // Always return the event even if there was an error. - return b, err + return false, err } // Interrupt the JS code if execution exceeds timeout. @@ -239,16 +239,16 @@ func (s *session) runProcessFunc(b *beat.Event) (out *beat.Event, err error) { if _, err = s.processFunc(goja.Undefined(), s.evt.JSObject()); err != nil { if s.tagOnException != "" { - mapstr.AddTags(b.Fields, []string{s.tagOnException}) + b.AddTags(s.tagOnException) } - appendString(b.Fields, "error.message", err.Error(), false) - return b, fmt.Errorf("failed in process function: %w", err) + appendString(b, "error.message", err.Error(), false) + return dropped, fmt.Errorf("failed in process function: %w", err) } if s.evt.IsCancelled() { - return nil, nil + return true, nil } - return b, nil + return false, nil } // Runtime returns the Javascript runtime used for this session. diff --git a/libbeat/processors/script/javascript/session_test.go b/libbeat/processors/script/javascript/session_test.go index 03d8a4236333..0900b86ca001 100644 --- a/libbeat/processors/script/javascript/session_test.go +++ b/libbeat/processors/script/javascript/session_test.go @@ -41,10 +41,11 @@ func TestSessionTagOnException(t *testing.T) { t.Fatal(err) } - evt, err := p.Run(testEvent()) + evt := testEvent() + _, err = p.Run(evt) assert.Error(t, err) - tags, _ := evt.GetValue("tags") + tags, _ := evt.GetValue(mapstr.TagsKey) assert.Equal(t, []string{"_js_exception"}, tags) } @@ -109,7 +110,7 @@ func TestSessionTestFunction(t *testing.T) { function test() { var event = process(new Event({"hello": "earth"})); - if (event.fields.hello !== "world") { + if (event.Get("hello") !== "world") { throw "invalid hello world"; } } @@ -149,8 +150,8 @@ func TestSessionTimeout(t *testing.T) { logp.TestingSetup() const runawayLoop = ` - while (!evt.fields.stop) { - evt.Put("hello", "world"); + while (!evt.Get("stop")) { + evt.Put("hello", "world"); } ` @@ -163,14 +164,14 @@ func TestSessionTimeout(t *testing.T) { t.Fatal(err) } - evt := &beat.Event{ + evt := beat.NewEventEditor(&beat.Event{ Fields: mapstr.M{ "stop": false, }, - } + }) // Execute and expect a timeout. - evt, err = p.Run(evt) + _, err = p.Run(evt) if assert.Error(t, err) { assert.Contains(t, err.Error(), timeoutError) @@ -189,7 +190,7 @@ func TestSessionTimeout(t *testing.T) { func TestSessionParallel(t *testing.T) { const script = ` - evt.Put("host.name", "workstation"); + evt.Put("host.name", "workstation"); ` p, err := NewFromConfig(Config{ @@ -210,11 +211,11 @@ func TestSessionParallel(t *testing.T) { go func() { defer wg.Done() for ctx.Err() == nil { - evt := &beat.Event{ + evt := beat.NewEventEditor(&beat.Event{ Fields: mapstr.M{ "host": mapstr.M{"name": "computer"}, }, - } + }) _, err := p.Run(evt) assert.NoError(t, err) }