[pkg/ottl] Add ParseSeverity function
#37280
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| # Use this changelog template to create an entry for release notes. | ||
|
|
||
| # One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' | ||
| change_type: enhancement | ||
|
|
||
| # The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) | ||
| component: pkg/ottl | ||
|
|
||
| # A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). | ||
| note: Add `ParseSeverity` function to define mappings for log severity levels. | ||
|
|
||
| # Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. | ||
| issues: [35778] | ||
|
|
||
| # (Optional) One or more lines of additional information to render under the primary note. | ||
| # These lines will be padded with 2 spaces and then inserted directly into the document. | ||
| # Use pipe (|) for multiline entries. | ||
| subtext: | ||
|
|
||
| # If your change doesn't affect end users or the exported elements of any package, | ||
| # you should instead start your pull request title with [chore] or use the "Skip Changelog" label. | ||
| # Optional: The change log or logs in which this entry should be included. | ||
| # e.g. '[user]' or '[user, api]' | ||
| # Include 'user' if the change is relevant to end users. | ||
| # Include 'api' if there is a change to a library API. | ||
| # Default: '[user]' | ||
| change_logs: [] |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,200 @@ | ||
| // Copyright The OpenTelemetry Authors | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| package ottlfuncs // import "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl/ottlfuncs" | ||
|
|
||
| import ( | ||
| "context" | ||
| "errors" | ||
| "fmt" | ||
|
|
||
| "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl" | ||
| ) | ||
|
|
||
| const ( | ||
| // http2xx is a special key that is represents a range from 200 to 299 | ||
| http2xx = "2xx" | ||
|
|
||
| // http3xx is a special key that is represents a range from 300 to 399 | ||
| http3xx = "3xx" | ||
|
|
||
| // http4xx is a special key that is represents a range from 400 to 499 | ||
| http4xx = "4xx" | ||
|
|
||
| // http5xx is a special key that is represents a range from 500 to 599 | ||
| http5xx = "5xx" | ||
|
|
||
| minKey = "min" | ||
| maxKey = "max" | ||
|
|
||
| rangeKey = "range" | ||
| equalsKey = "equals" | ||
| ) | ||
|
|
||
| type ParseSeverityArguments[K any] struct { | ||
| Target ottl.Getter[K] | ||
| Mapping ottl.PMapGetter[K] | ||
| } | ||
|
|
||
| func NewParseSeverityFactory[K any]() ottl.Factory[K] { | ||
| return ottl.NewFactory("ParseSeverity", &ParseSeverityArguments[K]{}, createParseSeverityFunction[K]) | ||
| } | ||
|
|
||
| func createParseSeverityFunction[K any](_ ottl.FunctionContext, oArgs ottl.Arguments) (ottl.ExprFunc[K], error) { | ||
| args, ok := oArgs.(*ParseSeverityArguments[K]) | ||
|
|
||
| if !ok { | ||
| return nil, errors.New("ParseSeverityFactory args must be of type *ParseSeverityArguments[K") | ||
| } | ||
|
|
||
| return parseSeverity[K](args.Target, args.Mapping), nil | ||
| } | ||
|
|
||
| func parseSeverity[K any](target ottl.Getter[K], mapping ottl.PMapGetter[K]) ottl.ExprFunc[K] { | ||
| return func(ctx context.Context, tCtx K) (any, error) { | ||
| severityMap, err := mapping.Get(ctx, tCtx) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("cannot get severity mapping: %w", err) | ||
| } | ||
|
|
||
| value, err := target.Get(ctx, tCtx) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("could not get log level: %w", err) | ||
| } | ||
|
|
||
| logLevel, err := evaluateSeverity(value, severityMap.AsRaw()) | ||
|
There was a problem hiding this comment. My main concern about this function is performance, building up/looping through the severity mappings for every single execution can be expensive. I wish we could enforce the That said, I'd try to avoid as many loops as possible, and use the There was a problem hiding this comment.
I fully agree with this part at least. This primary principle behind the stanza implementation is to build the mapping once and then having constant lookup times while processing each record. If we can't make this happen somehow in OTTL then we will pay quite a cost. Can we build this mapping inside of func parseSeverity[K any](target ottl.Getter[K], mapping ottl.PMapGetter[K]) ottl.ExprFunc[K] {
// build static map of severities once
return func(ctx context.Context, tCtx K) (any, error) {
// use static map in every execution
}There was a problem hiding this comment. Maybe there's a better way but if nothing else, I think we could declare the compiled mapping as a var within func parseSeverity[K any](target ottl.Getter[K], mapping ottl.PMapGetter[K]) ottl.ExprFunc[K] {
var compileOnce sync.Once
var staticMapping map[string]int
return func(ctx context.Context, tCtx K) (any, error) {
compileOnce.Do(func() {
precompiledMapping, getMappingErr = args.Mapping.Get(ctx, tCtx)
})
value, err := target.Get(ctx, tCtx)
if err != nil {
return nil, fmt.Errorf("could not get log level: %w", err)
}
sev, ok := staticMapping[value]
if !ok {
return defaultSeverity
}
return sev
}
}There was a problem hiding this comment. My favorite solution is finding a way to force the input to be a literal. It will take OTTL changes, but that is really what we want. There was a problem hiding this comment. Is it even theoretically possible for a range to be expressed as a literal? Granted, it's not strictly necessary that we support ranges but there are some use cases where it is quite convenient. There was a problem hiding this comment.
Yes I think it's possible, not exactly the range, but the whole map as literal, which for OTTL essentially means that the function's parameter value can be retrieved at the bootstrap time, and differently from the There was a problem hiding this comment.
I agree supporting literal inputs is nice but my point is that sometimes it's more user friendly to support something which can be interpreted once when the function is built. I think the notion of a range is one of those situations because e.g. no one wants to have to list all the HTTP status codes in order to assign them to severity levels. This shouldn't be a choice between literal inputs and recomputing a mapping for every context. If we can provide more user friendly inputs AND compute a complete mapping only once, this is better than literal inputs and also better than recomputing the same thing repeatedly. There was a problem hiding this comment.
I might be missing the point, but that's what literals should do. The term "literals" is somehow confusing here, for OTTL it means we have a function's parameter that is not a getter, instead, it's a raw immutable value that is available when the function is built. To support this use case, for example, OTTL needs to be changed so it knowns how to parse inputs like log_statements:
- context: log
statements:
- set(severity_number, ParseSeverity(severity_number, { "error":["err", { "min": 3, "max": 4 }]})) # Would work as the argument value is a literal, and cannot be changed by other statements.
- set(cache["mappings"], { "error":["err", { "min": 3, "max": 4 }]})
- set(severity_number, ParseSeverity(severity_number, cache["mappings"])) # Wouldn't work, as the cache["mappings"] path is mutable (getter), so it's not a literal and needs to be evaluate in every execution.
- set(cache["mappings"], { "error":["err", { "min": 5, "max": 6 }]})
There was a problem hiding this comment. I agree with your description of literals and agree we need them. My point though is about what happens after we have parsed the syntax. Every single line of Severity ranges tend to have a very reasonable and finite number of possible values so there's no need to evaluate logic every time we see a log. "Is this a range criteria?" "Is there a min?" "What is the min?", "Is this value greater than the min?", etc are all unnecessary if we can precompile this into a lookup table that is instant access. There was a problem hiding this comment. Oh I see, my answer was more focused on the OTTL side and how we could get the map parsed as literal. |
||
| if err != nil { | ||
| return nil, fmt.Errorf("could not map log level: %w", err) | ||
| } | ||
|
|
||
| return logLevel, nil | ||
| } | ||
| } | ||
|
|
||
| func evaluateSeverity(value any, severities map[string]any) (string, error) { | ||
| for level, criteria := range severities { | ||
| criteriaList, ok := criteria.([]any) | ||
| if !ok { | ||
| return "", errors.New("criteria for mapping log level must be []any") | ||
| } | ||
| match, err := evaluateSeverityMapping(value, criteriaList) | ||
| if err != nil { | ||
| return "", fmt.Errorf("could not evaluate log level of value '%v': %w", value, err) | ||
| } | ||
| if match { | ||
| return level, nil | ||
| } | ||
| } | ||
| return "", fmt.Errorf("no matching log level found for value '%v'", value) | ||
| } | ||
|
|
||
| func evaluateSeverityMapping(value any, criteria []any) (bool, error) { | ||
| switch v := value.(type) { | ||
| case string: | ||
| return evaluateSeverityStringMapping(v, criteria), nil | ||
| case int64: | ||
| return evaluateSeverityNumberMapping(v, criteria) | ||
| default: | ||
| return false, fmt.Errorf("log level must be either string or int64, but got %T", v) | ||
| } | ||
| } | ||
|
|
||
| func evaluateSeverityNumberMapping(value int64, criteria []any) (bool, error) { | ||
| for _, crit := range criteria { | ||
| criteriaItem, ok := crit.(map[string]any) | ||
| if !ok { | ||
| continue | ||
| } | ||
|
|
||
| // right now, we only have a "range" criteria for numeric log levels, so we specifically check for this here | ||
| rangeMapObj, ok := criteriaItem[rangeKey] | ||
| if !ok { | ||
| continue | ||
| } | ||
|
|
||
| // if we have a numeric severity number, we need to match with number ranges | ||
| rangeMap, ok := rangeMapObj.(map[string]any) | ||
| if !ok { | ||
| rangeMap, ok = parseValueRangePlaceholder(rangeMapObj) | ||
| if !ok { | ||
| continue | ||
| } | ||
| } | ||
| rangeMin, gotMin := rangeMap[minKey] | ||
| rangeMax, gotMax := rangeMap[maxKey] | ||
| if !gotMin || !gotMax { | ||
| return false, errors.New("range criteria must contain min and max values") | ||
| } | ||
| rangeMinInt, ok := rangeMin.(int64) | ||
| if !ok { | ||
| return false, fmt.Errorf("min must be int64, but got %T", rangeMin) | ||
| } | ||
| rangeMaxInt, ok := rangeMax.(int64) | ||
| if !ok { | ||
| return false, fmt.Errorf("max must be int64, but got %T", rangeMax) | ||
| } | ||
|
|
||
| if rangeMinInt <= value && rangeMaxInt >= value { | ||
| return true, nil | ||
| } | ||
| } | ||
| return false, nil | ||
| } | ||
|
|
||
| func parseValueRangePlaceholder(crit any) (map[string]any, bool) { | ||
| placeholder, ok := crit.(string) | ||
| if !ok { | ||
| return nil, false | ||
| } | ||
|
|
||
| switch placeholder { | ||
| case http2xx: | ||
| return map[string]any{ | ||
| "min": int64(200), | ||
| "max": int64(299), | ||
| }, true | ||
| case http3xx: | ||
| return map[string]any{ | ||
| "min": int64(300), | ||
| "max": int64(399), | ||
| }, true | ||
| case http4xx: | ||
| return map[string]any{ | ||
| "min": int64(400), | ||
| "max": int64(499), | ||
| }, true | ||
| case http5xx: | ||
| return map[string]any{ | ||
| "min": int64(500), | ||
| "max": int64(599), | ||
| }, true | ||
| default: | ||
| return nil, false | ||
| } | ||
| } | ||
|
|
||
| func evaluateSeverityStringMapping(value string, criteria []any) bool { | ||
| for _, crit := range criteria { | ||
| criteriaItem, ok := crit.(map[string]any) | ||
| if !ok { | ||
| return false | ||
| } | ||
| criteriaEquals, ok := criteriaItem[equalsKey] | ||
| if !ok { | ||
| return false | ||
| } | ||
|
|
||
| equalsObjs, ok := criteriaEquals.([]any) | ||
| if !ok { | ||
| return false | ||
| } | ||
| for _, equals := range equalsObjs { | ||
| if equalsStr, ok := equals.(string); ok { | ||
| if equalsStr == value { | ||
| return true | ||
| } | ||
| } | ||
| } | ||
| } | ||
| return false | ||
| } | ||
Uh oh!
There was an error while loading. Please reload this page.