diff --git a/.chloggen/indexing-pkg-ottl.yaml b/.chloggen/indexing-pkg-ottl.yaml new file mode 100644 index 000000000000..678525bfac99 --- /dev/null +++ b/.chloggen/indexing-pkg-ottl.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: breaking + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: pkg/ottl + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: "Support dynamic indexing of maps and slices." + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [36644] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] diff --git a/pkg/ottl/contexts/internal/map.go b/pkg/ottl/contexts/internal/map.go index 34e891c032c1..9965d6dc5c86 100644 --- a/pkg/ottl/contexts/internal/map.go +++ b/pkg/ottl/contexts/internal/map.go @@ -22,7 +22,11 @@ func GetMapValue[K any](ctx context.Context, tCtx K, m pcommon.Map, keys []ottl. return nil, err } if s == nil { - return nil, fmt.Errorf("non-string indexing is not supported") + resString, err := FetchValueFromExpression[K, string](ctx, tCtx, keys[0]) + if err != nil { + return nil, fmt.Errorf("unable to resolve a string index in map: %w", err) + } + s = resString } val, ok := m.Get(*s) @@ -43,7 +47,11 @@ func SetMapValue[K any](ctx context.Context, tCtx K, m pcommon.Map, keys []ottl. return err } if s == nil { - return fmt.Errorf("non-string indexing is not supported") + resString, err := FetchValueFromExpression[K, string](ctx, tCtx, keys[0]) + if err != nil { + return fmt.Errorf("unable to resolve a string index in map: %w", err) + } + s = resString } currentValue, ok := m.Get(*s) @@ -52,3 +60,22 @@ func SetMapValue[K any](ctx context.Context, tCtx K, m pcommon.Map, keys []ottl. } return setIndexableValue[K](ctx, tCtx, currentValue, val, keys[1:]) } + +func FetchValueFromExpression[K any, T int64 | string](ctx context.Context, tCtx K, key ottl.Key[K]) (*T, error) { + p, err := key.ExpressionGetter(ctx, tCtx) + if err != nil { + return nil, err + } + if p == nil { + return nil, fmt.Errorf("invalid key type") + } + res, err := p.Get(ctx, tCtx) + if err != nil { + return nil, err + } + resVal, ok := res.(T) + if !ok { + return nil, fmt.Errorf("could not resolve key for map/slice, expecting '%T' but got '%T'", resVal, res) + } + return &resVal, nil +} diff --git a/pkg/ottl/contexts/internal/map_test.go b/pkg/ottl/contexts/internal/map_test.go index 18d41462413e..cc1544e18e5e 100644 --- a/pkg/ottl/contexts/internal/map_test.go +++ b/pkg/ottl/contexts/internal/map_test.go @@ -16,6 +16,14 @@ import ( ) func Test_GetMapValue_Invalid(t *testing.T) { + getSetter := &ottl.StandardGetSetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return nil, nil + }, + Setter: func(_ context.Context, _ any, _ any) error { + return nil + }, + } tests := []struct { name string keys []ottl.Key[any] @@ -26,42 +34,49 @@ func Test_GetMapValue_Invalid(t *testing.T) { keys: []ottl.Key[any]{ &TestKey[any]{ I: ottltest.Intp(0), + G: getSetter, }, }, - err: fmt.Errorf("non-string indexing is not supported"), + err: fmt.Errorf("unable to resolve a string index in map: could not resolve key for map/slice, expecting 'string' but got ''"), }, { name: "index map with int", keys: []ottl.Key[any]{ &TestKey[any]{ S: ottltest.Strp("map"), + G: getSetter, }, &TestKey[any]{ I: ottltest.Intp(0), + G: getSetter, }, }, - err: fmt.Errorf("map must be indexed by a string"), + err: fmt.Errorf("unable to resolve a string index in map: could not resolve key for map/slice, expecting 'string' but got ''"), }, { name: "index slice with string", keys: []ottl.Key[any]{ &TestKey[any]{ S: ottltest.Strp("slice"), + G: getSetter, }, &TestKey[any]{ S: ottltest.Strp("invalid"), + G: getSetter, }, }, - err: fmt.Errorf("slice must be indexed by an int"), + err: fmt.Errorf("unable to resolve an integer index in slice: could not resolve key for map/slice, expecting 'int64' but got ''"), }, { name: "index too large", keys: []ottl.Key[any]{ &TestKey[any]{ S: ottltest.Strp("slice"), + G: getSetter, }, &TestKey[any]{ I: ottltest.Intp(1), + G: getSetter, }, }, err: fmt.Errorf("index 1 out of bounds"), @@ -71,9 +86,11 @@ func Test_GetMapValue_Invalid(t *testing.T) { keys: []ottl.Key[any]{ &TestKey[any]{ S: ottltest.Strp("slice"), + G: getSetter, }, &TestKey[any]{ I: ottltest.Intp(-1), + G: getSetter, }, }, err: fmt.Errorf("index -1 out of bounds"), @@ -83,9 +100,11 @@ func Test_GetMapValue_Invalid(t *testing.T) { keys: []ottl.Key[any]{ &TestKey[any]{ S: ottltest.Strp("string"), + G: getSetter, }, &TestKey[any]{ S: ottltest.Strp("string"), + G: getSetter, }, }, err: fmt.Errorf("type Str does not support string indexing"), @@ -102,7 +121,7 @@ func Test_GetMapValue_Invalid(t *testing.T) { s.AppendEmpty() _, err := GetMapValue[any](context.Background(), nil, m, tt.keys) - assert.Equal(t, tt.err, err) + assert.Equal(t, tt.err.Error(), err.Error()) }) } } @@ -129,6 +148,14 @@ func Test_GetMapValue_NilKey(t *testing.T) { } func Test_SetMapValue_Invalid(t *testing.T) { + getSetter := &ottl.StandardGetSetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return nil, nil + }, + Setter: func(_ context.Context, _ any, _ any) error { + return nil + }, + } tests := []struct { name string keys []ottl.Key[any] @@ -139,42 +166,49 @@ func Test_SetMapValue_Invalid(t *testing.T) { keys: []ottl.Key[any]{ &TestKey[any]{ I: ottltest.Intp(0), + G: getSetter, }, }, - err: fmt.Errorf("non-string indexing is not supported"), + err: fmt.Errorf("unable to resolve a string index in map: could not resolve key for map/slice, expecting 'string' but got ''"), }, { name: "index map with int", keys: []ottl.Key[any]{ &TestKey[any]{ S: ottltest.Strp("map"), + G: getSetter, }, &TestKey[any]{ I: ottltest.Intp(0), + G: getSetter, }, }, - err: fmt.Errorf("map must be indexed by a string"), + err: fmt.Errorf("unable to resolve a string index in map: could not resolve key for map/slice, expecting 'string' but got ''"), }, { name: "index slice with string", keys: []ottl.Key[any]{ &TestKey[any]{ S: ottltest.Strp("slice"), + G: getSetter, }, &TestKey[any]{ S: ottltest.Strp("map"), + G: getSetter, }, }, - err: fmt.Errorf("slice must be indexed by an int"), + err: fmt.Errorf("unable to resolve an integer index in slice: could not resolve key for map/slice, expecting 'int64' but got ''"), }, { name: "slice index too large", keys: []ottl.Key[any]{ &TestKey[any]{ S: ottltest.Strp("slice"), + G: getSetter, }, &TestKey[any]{ I: ottltest.Intp(1), + G: getSetter, }, }, err: fmt.Errorf("index 1 out of bounds"), @@ -184,9 +218,11 @@ func Test_SetMapValue_Invalid(t *testing.T) { keys: []ottl.Key[any]{ &TestKey[any]{ S: ottltest.Strp("slice"), + G: getSetter, }, &TestKey[any]{ I: ottltest.Intp(-1), + G: getSetter, }, }, err: fmt.Errorf("index -1 out of bounds"), @@ -196,9 +232,11 @@ func Test_SetMapValue_Invalid(t *testing.T) { keys: []ottl.Key[any]{ &TestKey[any]{ S: ottltest.Strp("string"), + G: getSetter, }, &TestKey[any]{ S: ottltest.Strp("string"), + G: getSetter, }, }, err: fmt.Errorf("type Str does not support string indexing"), @@ -215,7 +253,7 @@ func Test_SetMapValue_Invalid(t *testing.T) { s.AppendEmpty() err := SetMapValue[any](context.Background(), nil, m, tt.keys, "value") - assert.Equal(t, tt.err, err) + assert.Equal(t, tt.err.Error(), err.Error()) }) } } diff --git a/pkg/ottl/contexts/internal/path.go b/pkg/ottl/contexts/internal/path.go index 954d14329646..ebbb12a6eaae 100644 --- a/pkg/ottl/contexts/internal/path.go +++ b/pkg/ottl/contexts/internal/path.go @@ -46,6 +46,7 @@ var _ ottl.Key[any] = &TestKey[any]{} type TestKey[K any] struct { S *string I *int64 + G ottl.Getter[K] } func (k *TestKey[K]) String(_ context.Context, _ K) (*string, error) { @@ -55,3 +56,7 @@ func (k *TestKey[K]) String(_ context.Context, _ K) (*string, error) { func (k *TestKey[K]) Int(_ context.Context, _ K) (*int64, error) { return k.I, nil } + +func (k *TestKey[K]) ExpressionGetter(_ context.Context, _ K) (ottl.Getter[K], error) { + return k.G, nil +} diff --git a/pkg/ottl/contexts/internal/slice.go b/pkg/ottl/contexts/internal/slice.go index 5a90e281a902..9004d08ae836 100644 --- a/pkg/ottl/contexts/internal/slice.go +++ b/pkg/ottl/contexts/internal/slice.go @@ -22,7 +22,11 @@ func GetSliceValue[K any](ctx context.Context, tCtx K, s pcommon.Slice, keys []o return nil, err } if i == nil { - return nil, fmt.Errorf("non-integer indexing is not supported") + resInt, err := FetchValueFromExpression[K, int64](ctx, tCtx, keys[0]) + if err != nil { + return nil, fmt.Errorf("unable to resolve an integer index in slice: %w", err) + } + i = resInt } idx := int(*i) @@ -44,7 +48,11 @@ func SetSliceValue[K any](ctx context.Context, tCtx K, s pcommon.Slice, keys []o return err } if i == nil { - return fmt.Errorf("non-integer indexing is not supported") + resInt, err := FetchValueFromExpression[K, int64](ctx, tCtx, keys[0]) + if err != nil { + return fmt.Errorf("unable to resolve an integer index in slice: %w", err) + } + i = resInt } idx := int(*i) diff --git a/pkg/ottl/contexts/internal/slice_test.go b/pkg/ottl/contexts/internal/slice_test.go index 0f238225251a..8e1b7e0b977b 100644 --- a/pkg/ottl/contexts/internal/slice_test.go +++ b/pkg/ottl/contexts/internal/slice_test.go @@ -16,6 +16,14 @@ import ( ) func Test_GetSliceValue_Invalid(t *testing.T) { + getSetter := &ottl.StandardGetSetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return nil, nil + }, + Setter: func(_ context.Context, _ any, _ any) error { + return nil + }, + } tests := []struct { name string keys []ottl.Key[any] @@ -26,15 +34,17 @@ func Test_GetSliceValue_Invalid(t *testing.T) { keys: []ottl.Key[any]{ &TestKey[any]{ S: ottltest.Strp("key"), + G: getSetter, }, }, - err: fmt.Errorf("non-integer indexing is not supported"), + err: fmt.Errorf(`unable to resolve an integer index in slice: could not resolve key for map/slice, expecting 'int64' but got ''`), }, { name: "index too large", keys: []ottl.Key[any]{ &TestKey[any]{ I: ottltest.Intp(1), + G: getSetter, }, }, err: fmt.Errorf("index 1 out of bounds"), @@ -44,6 +54,7 @@ func Test_GetSliceValue_Invalid(t *testing.T) { keys: []ottl.Key[any]{ &TestKey[any]{ I: ottltest.Intp(-1), + G: getSetter, }, }, err: fmt.Errorf("index -1 out of bounds"), @@ -53,9 +64,11 @@ func Test_GetSliceValue_Invalid(t *testing.T) { keys: []ottl.Key[any]{ &TestKey[any]{ I: ottltest.Intp(0), + G: getSetter, }, &TestKey[any]{ S: ottltest.Strp("string"), + G: getSetter, }, }, err: fmt.Errorf("type Str does not support string indexing"), @@ -68,7 +81,7 @@ func Test_GetSliceValue_Invalid(t *testing.T) { s.AppendEmpty().SetStr("val") _, err := GetSliceValue[any](context.Background(), nil, s, tt.keys) - assert.Equal(t, tt.err, err) + assert.Equal(t, tt.err.Error(), err.Error()) }) } } @@ -79,6 +92,14 @@ func Test_GetSliceValue_NilKey(t *testing.T) { } func Test_SetSliceValue_Invalid(t *testing.T) { + getSetter := &ottl.StandardGetSetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return nil, nil + }, + Setter: func(_ context.Context, _ any, _ any) error { + return nil + }, + } tests := []struct { name string keys []ottl.Key[any] @@ -89,15 +110,17 @@ func Test_SetSliceValue_Invalid(t *testing.T) { keys: []ottl.Key[any]{ &TestKey[any]{ S: ottltest.Strp("key"), + G: getSetter, }, }, - err: fmt.Errorf("non-integer indexing is not supported"), + err: fmt.Errorf(`unable to resolve an integer index in slice: could not resolve key for map/slice, expecting 'int64' but got ''`), }, { name: "index too large", keys: []ottl.Key[any]{ &TestKey[any]{ I: ottltest.Intp(1), + G: getSetter, }, }, err: fmt.Errorf("index 1 out of bounds"), @@ -107,6 +130,7 @@ func Test_SetSliceValue_Invalid(t *testing.T) { keys: []ottl.Key[any]{ &TestKey[any]{ I: ottltest.Intp(-1), + G: getSetter, }, }, err: fmt.Errorf("index -1 out of bounds"), @@ -116,9 +140,11 @@ func Test_SetSliceValue_Invalid(t *testing.T) { keys: []ottl.Key[any]{ &TestKey[any]{ I: ottltest.Intp(0), + G: getSetter, }, &TestKey[any]{ S: ottltest.Strp("string"), + G: getSetter, }, }, err: fmt.Errorf("type Str does not support string indexing"), @@ -131,7 +157,7 @@ func Test_SetSliceValue_Invalid(t *testing.T) { s.AppendEmpty().SetStr("val") err := SetSliceValue[any](context.Background(), nil, s, tt.keys, "value") - assert.Equal(t, tt.err, err) + assert.Equal(t, tt.err.Error(), err.Error()) }) } } diff --git a/pkg/ottl/contexts/internal/value.go b/pkg/ottl/contexts/internal/value.go index ce335854eb3a..34136fe89caf 100644 --- a/pkg/ottl/contexts/internal/value.go +++ b/pkg/ottl/contexts/internal/value.go @@ -71,27 +71,35 @@ func SetValue(value pcommon.Value, val any) error { func getIndexableValue[K any](ctx context.Context, tCtx K, value pcommon.Value, keys []ottl.Key[K]) (any, error) { val := value var ok bool - for i := 0; i < len(keys); i++ { + for index := 0; index < len(keys); index++ { switch val.Type() { case pcommon.ValueTypeMap: - s, err := keys[i].String(ctx, tCtx) + s, err := keys[index].String(ctx, tCtx) if err != nil { return nil, err } if s == nil { - return nil, fmt.Errorf("map must be indexed by a string") + resString, err := FetchValueFromExpression[K, string](ctx, tCtx, keys[index]) + if err != nil { + return nil, fmt.Errorf("unable to resolve a string index in map: %w", err) + } + s = resString } val, ok = val.Map().Get(*s) if !ok { return nil, nil } case pcommon.ValueTypeSlice: - i, err := keys[i].Int(ctx, tCtx) + i, err := keys[index].Int(ctx, tCtx) if err != nil { return nil, err } if i == nil { - return nil, fmt.Errorf("slice must be indexed by an int") + resInt, err := FetchValueFromExpression[K, int64](ctx, tCtx, keys[index]) + if err != nil { + return nil, fmt.Errorf("unable to resolve an integer index in slice: %w", err) + } + i = resInt } if int(*i) >= val.Slice().Len() || int(*i) < 0 { return nil, fmt.Errorf("index %v out of bounds", *i) @@ -117,15 +125,19 @@ func setIndexableValue[K any](ctx context.Context, tCtx K, currentValue pcommon. return err } - for i := 0; i < len(keys); i++ { + for index := 0; index < len(keys); index++ { switch currentValue.Type() { case pcommon.ValueTypeMap: - s, err := keys[i].String(ctx, tCtx) + s, err := keys[index].String(ctx, tCtx) if err != nil { return err } if s == nil { - return errors.New("map must be indexed by a string") + resString, err := FetchValueFromExpression[K, string](ctx, tCtx, keys[index]) + if err != nil { + return fmt.Errorf("unable to resolve a string index in map: %w", err) + } + s = resString } potentialValue, ok := currentValue.Map().Get(*s) if !ok { @@ -134,23 +146,27 @@ func setIndexableValue[K any](ctx context.Context, tCtx K, currentValue pcommon. currentValue = potentialValue } case pcommon.ValueTypeSlice: - i, err := keys[i].Int(ctx, tCtx) + i, err := keys[index].Int(ctx, tCtx) if err != nil { return err } if i == nil { - return errors.New("slice must be indexed by an int") + resInt, err := FetchValueFromExpression[K, int64](ctx, tCtx, keys[index]) + if err != nil { + return fmt.Errorf("unable to resolve an integer index in slice: %w", err) + } + i = resInt } if int(*i) >= currentValue.Slice().Len() || int(*i) < 0 { return fmt.Errorf("index %v out of bounds", *i) } currentValue = currentValue.Slice().At(int(*i)) case pcommon.ValueTypeEmpty: - s, err := keys[i].String(ctx, tCtx) + s, err := keys[index].String(ctx, tCtx) if err != nil { return err } - i, err := keys[i].Int(ctx, tCtx) + i, err := keys[index].Int(ctx, tCtx) if err != nil { return err } @@ -164,7 +180,20 @@ func setIndexableValue[K any](ctx context.Context, tCtx K, currentValue pcommon. } currentValue = currentValue.Slice().AppendEmpty() default: - return errors.New("neither a string nor an int index was given, this is an error in the OTTL") + resString, errString := FetchValueFromExpression[K, string](ctx, tCtx, keys[index]) + resInt, errInt := FetchValueFromExpression[K, int64](ctx, tCtx, keys[index]) + switch { + case errInt == nil: + currentValue.SetEmptySlice() + for k := 0; k < int(*resInt); k++ { + currentValue.Slice().AppendEmpty() + } + currentValue = currentValue.Slice().AppendEmpty() + case errString == nil: + currentValue = currentValue.SetEmptyMap().PutEmpty(*resString) + default: + return errors.New("neither a string nor an int index was given, this is an error in the OTTL") + } } default: return fmt.Errorf("type %v does not support string indexing", currentValue.Type()) diff --git a/pkg/ottl/contexts/internal/value_test.go b/pkg/ottl/contexts/internal/value_test.go index 5be89f72ea46..726fd6de5eaa 100644 --- a/pkg/ottl/contexts/internal/value_test.go +++ b/pkg/ottl/contexts/internal/value_test.go @@ -13,10 +13,10 @@ import ( "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl" ) -func Test_SetIndexableValue_EmptyValueNoIndex(t *testing.T) { +func Test_SetIndexableValue_InvalidValue(t *testing.T) { keys := []ottl.Key[any]{ &TestKey[any]{}, } - err := setIndexableValue[any](context.Background(), nil, pcommon.NewValueEmpty(), nil, keys) + err := setIndexableValue[any](context.Background(), nil, pcommon.NewValueStr("str"), nil, keys) assert.Error(t, err) } diff --git a/pkg/ottl/e2e/e2e_test.go b/pkg/ottl/e2e/e2e_test.go index 72077e66dd23..a11d78a72305 100644 --- a/pkg/ottl/e2e/e2e_test.go +++ b/pkg/ottl/e2e/e2e_test.go @@ -308,10 +308,10 @@ func Test_e2e_editors(t *testing.T) { logStatements, err := logParser.ParseStatement(tt.statement) assert.NoError(t, err) - tCtx := constructLogTransformContext() + tCtx := constructLogTransformContextEditors() _, _, _ = logStatements.Execute(context.Background(), tCtx) - exTCtx := constructLogTransformContext() + exTCtx := constructLogTransformContextEditors() tt.want(exTCtx) assert.NoError(t, plogtest.CompareResourceLogs(newResourceLogs(exTCtx), newResourceLogs(tCtx))) @@ -323,7 +323,88 @@ func Test_e2e_converters(t *testing.T) { tests := []struct { statement string want func(tCtx ottllog.TransformContext) + wantErr bool + errMsg string }{ + { + statement: `set(attributes["newOne"], attributes[1])`, + want: func(_ ottllog.TransformContext) {}, + errMsg: "unable to resolve a string index in map: invalid key type", + }, + { + statement: `set(attributes["array"][0.0], "bar")`, + want: func(_ ottllog.TransformContext) {}, + errMsg: "unable to resolve an integer index in slice: invalid key type", + }, + { + statement: `set(attributes["array"][ConvertCase(attributes["A|B|C"], "upper")], "bar")`, + want: func(_ ottllog.TransformContext) {}, + errMsg: "unable to resolve an integer index in slice: could not resolve key for map/slice, expecting 'int64'", + }, + { + statement: `set(attributes[ConvertCase(attributes["A|B|C"], "upper")], "myvalue")`, + want: func(tCtx ottllog.TransformContext) { + tCtx.GetLogRecord().Attributes().PutStr("SOMETHING", "myvalue") + }, + }, + { + statement: `set(attributes[ConvertCase(attributes[attributes["flags"]], "upper")], "myvalue")`, + want: func(tCtx ottllog.TransformContext) { + tCtx.GetLogRecord().Attributes().PutStr("SOMETHING", "myvalue") + }, + }, + { + statement: `set(attributes[attributes["flags"]], "something33")`, + want: func(tCtx ottllog.TransformContext) { + tCtx.GetLogRecord().Attributes().PutStr("A|B|C", "something33") + }, + }, + { + statement: `set(attributes[attributes[attributes["flags"]]], "something2")`, + want: func(tCtx ottllog.TransformContext) { + tCtx.GetLogRecord().Attributes().PutStr("something", "something2") + }, + }, + { + statement: `set(body, attributes[attributes["foo"][attributes["slice"]][attributes["int_value"]]])`, + want: func(tCtx ottllog.TransformContext) { + tCtx.GetLogRecord().Body().SetStr("val2") + }, + }, + { + statement: `set(resource.attributes[attributes["flags"]], "something33")`, + want: func(tCtx ottllog.TransformContext) { + tCtx.GetResource().Attributes().PutStr("A|B|C", "something33") + }, + }, + { + statement: `set(resource.attributes[resource.attributes[attributes["flags"]]], "something33")`, + want: func(tCtx ottllog.TransformContext) { + tCtx.GetResource().Attributes().PutStr("newValue", "something33") + }, + }, + { + statement: `set(attributes[resource.attributes[attributes["flags"]]], "something33")`, + want: func(tCtx ottllog.TransformContext) { + tCtx.GetLogRecord().Attributes().PutStr("newValue", "something33") + }, + }, + { + statement: `set(body, attributes["array"])`, + want: func(tCtx ottllog.TransformContext) { + arr := tCtx.GetLogRecord().Body().SetEmptySlice() + arr0 := arr.AppendEmpty() + arr0.SetStr("looong") + }, + }, + { + statement: `set(attributes["array"][attributes["int_value"]], 3)`, + want: func(tCtx ottllog.TransformContext) { + arr := tCtx.GetLogRecord().Attributes().PutEmptySlice("array") + arr0 := arr.AppendEmpty() + arr0.SetInt(3) + }, + }, { statement: `set(attributes["test"], Base64Decode("cGFzcw=="))`, want: func(tCtx ottllog.TransformContext) { @@ -965,7 +1046,12 @@ func Test_e2e_converters(t *testing.T) { assert.NoError(t, err) tCtx := constructLogTransformContext() - _, _, _ = logStatements.Execute(context.Background(), tCtx) + _, _, err = logStatements.Execute(context.Background(), tCtx) + if tt.errMsg == "" { + assert.NoError(t, err) + } else { + assert.Contains(t, err.Error(), tt.errMsg) + } exTCtx := constructLogTransformContext() tt.want(exTCtx) @@ -993,6 +1079,13 @@ func Test_e2e_ottl_features(t *testing.T) { tCtx.GetLogRecord().Attributes().PutStr("test", "pass") }, }, + { + name: "where clause with dynamic indexing", + statement: `set(attributes["foo"], "bar") where attributes[attributes["flags"]] != nil`, + want: func(tCtx ottllog.TransformContext) { + tCtx.GetLogRecord().Attributes().PutStr("foo", "bar") + }, + }, { name: "Using enums", statement: `set(severity_number, SEVERITY_NUMBER_TRACE2) where severity_number == SEVERITY_NUMBER_TRACE`, @@ -1125,6 +1218,57 @@ func Test_ProcessTraces_TraceContext(t *testing.T) { func constructLogTransformContext() ottllog.TransformContext { resource := pcommon.NewResource() resource.Attributes().PutStr("host.name", "localhost") + resource.Attributes().PutStr("A|B|C", "newValue") + + scope := pcommon.NewInstrumentationScope() + scope.SetName("scope") + + logRecord := plog.NewLogRecord() + logRecord.Body().SetStr("operationA") + logRecord.SetTimestamp(TestLogTimestamp) + logRecord.SetObservedTimestamp(TestObservedTimestamp) + logRecord.SetDroppedAttributesCount(1) + logRecord.SetFlags(plog.DefaultLogRecordFlags.WithIsSampled(true)) + logRecord.SetSeverityNumber(1) + logRecord.SetTraceID(traceID) + logRecord.SetSpanID(spanID) + logRecord.Attributes().PutStr("http.method", "get") + logRecord.Attributes().PutStr("http.path", "/health") + logRecord.Attributes().PutStr("http.url", "http://localhost/health") + logRecord.Attributes().PutStr("flags", "A|B|C") + logRecord.Attributes().PutStr("total.string", "123456789") + logRecord.Attributes().PutStr("A|B|C", "something") + logRecord.Attributes().PutStr("foo", "foo") + logRecord.Attributes().PutStr("slice", "slice") + logRecord.Attributes().PutStr("val", "val2") + logRecord.Attributes().PutInt("int_value", 0) + arr := logRecord.Attributes().PutEmptySlice("array") + arr0 := arr.AppendEmpty() + arr0.SetStr("looong") + m := logRecord.Attributes().PutEmptyMap("foo") + m.PutStr("bar", "pass") + m.PutStr("flags", "pass") + s := m.PutEmptySlice("slice") + v := s.AppendEmpty() + v.SetStr("val") + m2 := m.PutEmptyMap("nested") + m2.PutStr("test", "pass") + + s2 := logRecord.Attributes().PutEmptySlice("things") + thing1 := s2.AppendEmpty().SetEmptyMap() + thing1.PutStr("name", "foo") + thing1.PutInt("value", 2) + + thing2 := s2.AppendEmpty().SetEmptyMap() + thing2.PutStr("name", "bar") + thing2.PutInt("value", 5) + + return ottllog.NewTransformContext(logRecord, scope, resource, plog.NewScopeLogs(), plog.NewResourceLogs()) +} + +func constructLogTransformContextEditors() ottllog.TransformContext { + resource := pcommon.NewResource() + resource.Attributes().PutStr("host.name", "localhost") scope := pcommon.NewInstrumentationScope() scope.SetName("scope") diff --git a/pkg/ottl/functions.go b/pkg/ottl/functions.go index 4ff92123c7e6..5740328fa1c2 100644 --- a/pkg/ottl/functions.go +++ b/pkg/ottl/functions.go @@ -53,6 +53,17 @@ func buildOriginalKeysText(keys []key) string { if k.String != nil { builder.WriteString(*k.String) } + if k.Expression != nil { + if k.Expression.Path != nil { + builder.WriteString(buildOriginalText(k.Expression.Path)) + } + if k.Expression.Float != nil { + builder.WriteString(strconv.FormatFloat(*k.Expression.Float, 'f', 10, 64)) + } + if k.Expression.Int != nil { + builder.WriteString(strconv.FormatInt(*k.Expression.Int, 10)) + } + } builder.WriteString("]") } } @@ -72,10 +83,14 @@ func (p *Parser[K]) newPath(path *path) (*basePath[K], error) { originalText := buildOriginalText(path) var current *basePath[K] for i := len(fields) - 1; i >= 0; i-- { + keys, err := p.newKeys(fields[i].Keys) + if err != nil { + return nil, err + } current = &basePath[K]{ context: pathContext, name: fields[i].Name, - keys: newKeys[K](fields[i].Keys), + keys: keys, nextPath: current, originalText: originalText, } @@ -203,18 +218,36 @@ func (p *basePath[K]) isComplete() error { return p.nextPath.isComplete() } -func newKeys[K any](keys []key) []Key[K] { +func (p *Parser[K]) newKeys(keys []key) ([]Key[K], error) { if len(keys) == 0 { - return nil + return nil, nil } ks := make([]Key[K], len(keys)) for i := range keys { + var getter Getter[K] + if keys[i].Expression != nil { + if keys[i].Expression.Path != nil { + g, err := p.buildGetSetterFromPath(keys[i].Expression.Path) + if err != nil { + return nil, err + } + getter = g + } + if keys[i].Expression.Converter != nil { + g, err := p.newGetterFromConverter(*keys[i].Expression.Converter) + if err != nil { + return nil, err + } + getter = g + } + } ks[i] = &baseKey[K]{ s: keys[i].String, i: keys[i].Int, + g: getter, } } - return ks + return ks, nil } // Key represents a chain of keys in an OTTL statement, such as `attributes["foo"]["bar"]`. @@ -230,6 +263,12 @@ type Key[K any] interface { // If the Key does not have a int value the returned value is nil. // If Key experiences an error retrieving the value it is returned. Int(context.Context, K) (*int64, error) + + // ExpressionGetter returns a Getter to the expression, that can be + // part of the path. + // If the Key does not have an expression the returned value is nil. + // If Key experiences an error retrieving the value it is returned. + ExpressionGetter(context.Context, K) (Getter[K], error) } var _ Key[any] = &baseKey[any]{} @@ -237,6 +276,7 @@ var _ Key[any] = &baseKey[any]{} type baseKey[K any] struct { s *string i *int64 + g Getter[K] } func (k *baseKey[K]) String(_ context.Context, _ K) (*string, error) { @@ -247,6 +287,10 @@ func (k *baseKey[K]) Int(_ context.Context, _ K) (*int64, error) { return k.i, nil } +func (k *baseKey[K]) ExpressionGetter(_ context.Context, _ K) (Getter[K], error) { + return k.g, nil +} + func (p *Parser[K]) parsePath(ip *basePath[K]) (GetSetter[K], error) { g, err := p.pathParser(ip) if err != nil { @@ -474,6 +518,18 @@ func (p *Parser[K]) buildSliceArg(argVal value, argType reflect.Type) (any, erro } } +func (p *Parser[K]) buildGetSetterFromPath(path *path) (GetSetter[K], error) { + np, err := p.newPath(path) + if err != nil { + return nil, err + } + arg, err := p.parsePath(np) + if err != nil { + return nil, err + } + return arg, nil +} + // Handle interfaces that can be passed as arguments to OTTL functions. func (p *Parser[K]) buildArg(argVal value, argType reflect.Type) (any, error) { name := argType.Name() @@ -481,18 +537,10 @@ func (p *Parser[K]) buildArg(argVal value, argType reflect.Type) (any, error) { case strings.HasPrefix(name, "Setter"): fallthrough case strings.HasPrefix(name, "GetSetter"): - if argVal.Literal == nil || argVal.Literal.Path == nil { - return nil, fmt.Errorf("must be a path") + if argVal.Literal != nil && argVal.Literal.Path != nil { + return p.buildGetSetterFromPath(argVal.Literal.Path) } - np, err := p.newPath(argVal.Literal.Path) - if err != nil { - return nil, err - } - arg, err := p.parsePath(np) - if err != nil { - return nil, err - } - return arg, nil + return nil, fmt.Errorf("must be a path") case strings.HasPrefix(name, "Getter"): arg, err := p.newGetter(argVal) if err != nil { diff --git a/pkg/ottl/functions_test.go b/pkg/ottl/functions_test.go index a7bd4aef87c7..bb81c316dd27 100644 --- a/pkg/ottl/functions_test.go +++ b/pkg/ottl/functions_test.go @@ -2525,6 +2525,13 @@ func Test_baseKey_Int(t *testing.T) { } func Test_newKey(t *testing.T) { + ps, _ := NewParser[any]( + defaultFunctionsForTests(), + testParsePath[any], + componenttest.NewNopTelemetrySettings(), + WithEnumParser[any](testParseEnum), + WithPathContextNames[any]([]string{"log"}), + ) keys := []key{ { String: ottltest.Strp("foo"), @@ -2533,7 +2540,7 @@ func Test_newKey(t *testing.T) { String: ottltest.Strp("bar"), }, } - ks := newKeys[any](keys) + ks, _ := ps.newKeys(keys) assert.Len(t, ks, 2) diff --git a/pkg/ottl/grammar.go b/pkg/ottl/grammar.go index b7d6743f4afa..8a79ca978db6 100644 --- a/pkg/ottl/grammar.go +++ b/pkg/ottl/grammar.go @@ -270,15 +270,35 @@ type path struct { Fields []field `parser:"@@ ( '.' @@ )*"` } +func (p *path) accept(v grammarVisitor) { + v.visitPath(p) + for _, field := range p.Fields { + field.accept(v) + } +} + // field is an item within a path. type field struct { Name string `parser:"@Lowercase"` Keys []key `parser:"( @@ )*"` } +func (f *field) accept(v grammarVisitor) { + for _, key := range f.Keys { + key.accept(v) + } +} + type key struct { - String *string `parser:"'[' (@String "` - Int *int64 `parser:"| @Int) ']'"` + String *string `parser:"'[' (@String "` + Int *int64 `parser:"| @Int"` + Expression *mathExprLiteral `parser:"| @@ ) ']'"` +} + +func (k *key) accept(v grammarVisitor) { + if k.Expression != nil { + k.Expression.accept(v) + } } type list struct { @@ -343,7 +363,7 @@ type mathExprLiteral struct { func (m *mathExprLiteral) accept(v grammarVisitor) { v.visitMathExprLiteral(m) if m.Path != nil { - v.visitPath(m.Path) + m.Path.accept(v) } if m.Editor != nil { m.Editor.accept(v) diff --git a/pkg/ottl/lexer_test.go b/pkg/ottl/lexer_test.go index b73d71d80e8f..4cd54f3fd9b8 100644 --- a/pkg/ottl/lexer_test.go +++ b/pkg/ottl/lexer_test.go @@ -130,6 +130,15 @@ func Test_lexer(t *testing.T) { {"String", `"bar"`}, {"RBrace", "}"}, }}, + {"Dynamic path", `attributes[attributes["foo"]]`, false, []result{ + {"Lowercase", "attributes"}, + {"Punct", "["}, + {"Lowercase", "attributes"}, + {"Punct", "["}, + {"String", `"foo"`}, + {"Punct", "]"}, + {"Punct", "]"}, + }}, } for _, tt := range tests {