From 1f0ba832f5b27b6390055f21a6b06ec872de4988 Mon Sep 17 00:00:00 2001 From: Phillip Adair Stewart Whelan Date: Tue, 6 May 2025 13:19:59 -0400 Subject: [PATCH 1/7] refactor to use goccy/go-yaml. Signed-off-by: Phillip Adair Stewart Whelan --- dump.go | 8 ++++---- go.mod | 1 + go.sum | 2 ++ plugin.go | 8 ++++---- property/yaml.go | 44 ++++++++++++++++---------------------------- 5 files changed, 27 insertions(+), 36 deletions(-) diff --git a/dump.go b/dump.go index 6a37862..17c2c44 100644 --- a/dump.go +++ b/dump.go @@ -7,7 +7,7 @@ import ( "io" "strings" - "gopkg.in/yaml.v3" + "github.com/goccy/go-yaml" ) var ErrFormatUnknown = errors.New("format unknown") @@ -40,8 +40,7 @@ func ParseAsClassic(raw string) (Config, error) { func ParseAsYAML(raw string) (Config, error) { var out Config - dec := yaml.NewDecoder(strings.NewReader(raw)) - dec.KnownFields(true) + dec := yaml.NewDecoder(strings.NewReader(raw), yaml.Strict()) err := dec.Decode(&out) if errors.Is(err, io.EOF) { return out, nil @@ -84,7 +83,8 @@ func (c Config) DumpAsClassic() (string, error) { } func (c Config) DumpAsYAML() (string, error) { - b, err := yaml.Marshal(c) + b, err := yaml.MarshalWithOptions(c, + yaml.Indent(4), yaml.IndentSequence(true), yaml.UseSingleQuote(true)) if err != nil { return "", err } diff --git a/go.mod b/go.mod index 64f0f23..c10da4e 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.23.0 toolchain go1.24.2 require ( + github.com/goccy/go-yaml v1.17.1 github.com/muesli/reflow v0.3.0 github.com/spf13/pflag v1.0.6 github.com/stretchr/testify v1.10.0 diff --git a/go.sum b/go.sum index f37694c..16ea91e 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/goccy/go-yaml v1.17.1 h1:LI34wktB2xEE3ONG/2Ar54+/HJVBriAGJ55PHls4YuY= +github.com/goccy/go-yaml v1.17.1/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= diff --git a/plugin.go b/plugin.go index a004d8d..ef70490 100644 --- a/plugin.go +++ b/plugin.go @@ -4,8 +4,8 @@ import ( "encoding/json" "fmt" + yaml "github.com/goccy/go-yaml" "golang.org/x/exp/slices" - "gopkg.in/yaml.v3" "github.com/calyptia/go-fluentbit-config/v2/property" ) @@ -50,9 +50,9 @@ func (plugins *Plugins) UnmarshalJSON(data []byte) error { return nil } -func (plugins *Plugins) UnmarshalYAML(node *yaml.Node) error { +func (plugins *Plugins) UnmarshalYAML(node []byte) error { var dest []Plugin - err := node.Decode(&dest) + err := yaml.Unmarshal(node, &dest) if err != nil { return err } @@ -101,7 +101,7 @@ func (p *Plugin) UnmarshalJSON(data []byte) error { return nil } -func (p *Plugin) UnmarshalYAML(node *yaml.Node) error { +func (p *Plugin) UnmarshalYAML(node []byte) error { err := p.Properties.UnmarshalYAML(node) if err != nil { return err diff --git a/property/yaml.go b/property/yaml.go index d0471c6..95720f8 100644 --- a/property/yaml.go +++ b/property/yaml.go @@ -3,7 +3,7 @@ package property import ( "fmt" - "gopkg.in/yaml.v3" + "github.com/goccy/go-yaml" ) // MarshalYAML implements yaml.Marshaler interface @@ -17,45 +17,33 @@ func (pp Properties) MarshalYAML() (any, error) { return []any{}, nil } - node := &yaml.Node{ - Kind: yaml.MappingNode, - } - - for _, p := range pp { - valueNode := &yaml.Node{} - if err := valueNode.Encode(p.Value); err != nil { - return nil, fmt.Errorf("yaml encode property value: %w", err) - } + properties := make(yaml.MapSlice, len(pp)) - node.Content = append(node.Content, &yaml.Node{ - Kind: yaml.ScalarNode, - Value: p.Key, - }, valueNode) + for pindex, prop := range pp { + properties[pindex] = yaml.MapItem{Key: prop.Key, Value: prop.Value} } - return node, nil + return properties, nil } // UnmarshalYAML implements yaml.Unmarshaler interface // to unmarshal an object into a sorted list of properties. -func (pp *Properties) UnmarshalYAML(node *yaml.Node) error { - d := len(node.Content) - if d%2 != 0 { - return fmt.Errorf("expected even items for key-value") +func (pp *Properties) UnmarshalYAML(node []byte) error { + var mprops yaml.MapSlice + + if err := yaml.Unmarshal(node, &mprops); err != nil { + return err } - for i := 0; i < d; i += 2 { + for _, mprop := range mprops { var prop Property + var ok bool - keyNode := node.Content[i] - if err := keyNode.Decode(&prop.Key); err != nil { - return fmt.Errorf("yaml decode property key: %w", err) - } - - valueNode := node.Content[i+1] - if err := valueNode.Decode(&prop.Value); err != nil { - return fmt.Errorf("yaml decode property value: %w", err) + prop.Key, ok = mprop.Key.(string) + if !ok { + return fmt.Errorf("unable to unmarshal key") } + prop.Value = mprop.Value *pp = append(*pp, prop) } From 66ee4c3f820ec6917ba3ed55fff7fa0ef099de7f Mon Sep 17 00:00:00 2001 From: Phillip Adair Stewart Whelan Date: Tue, 6 May 2025 13:21:14 -0400 Subject: [PATCH 2/7] validator_test: update to use goccy/go-yaml. Signed-off-by: Phillip Adair Stewart Whelan --- validator_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator_test.go b/validator_test.go index a97ef26..1d260a2 100644 --- a/validator_test.go +++ b/validator_test.go @@ -5,7 +5,7 @@ import ( "testing" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v3" + yaml "github.com/goccy/go-yaml" ) func TestConfig_Validate(t *testing.T) { From cbb902ec2aca968b6a1e55d503e339b36d1de3e1 Mon Sep 17 00:00:00 2001 From: Phillip Adair Stewart Whelan Date: Tue, 6 May 2025 13:21:46 -0400 Subject: [PATCH 3/7] config_test: update to use goccy/go-yaml, fix indentation in test. Signed-off-by: Phillip Adair Stewart Whelan --- config_test.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/config_test.go b/config_test.go index c2708c0..f638bfe 100644 --- a/config_test.go +++ b/config_test.go @@ -10,7 +10,7 @@ import ( "testing" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v3" + yaml "github.com/goccy/go-yaml" "github.com/calyptia/go-fluentbit-config/v2/property" ) @@ -495,11 +495,11 @@ func Example_configWithProcessors() { // inputs: // - name: dummy // processors: - // logs: - // - name: calyptia - // actions: - // type: block_keys - // opts: - // regex: star - // regexEngine: pcre2 + // logs: + // - name: calyptia + // actions: + // type: block_keys + // opts: + // regex: star + // regexEngine: pcre2 } From a35962b48228d5fafa3c526ae281d74d9f384b7e Mon Sep 17 00:00:00 2001 From: Phillip Adair Stewart Whelan Date: Tue, 6 May 2025 13:23:12 -0400 Subject: [PATCH 4/7] testdata: fix indentation to strictly use 4 spaces. Signed-off-by: Phillip Adair Stewart Whelan --- testdata/array-value.yaml | 4 +-- testdata/full.yaml | 4 +-- testdata/multiline-value.yaml | 4 +-- testdata/processors/processors.yaml | 40 ++++++++++++++--------------- 4 files changed, 26 insertions(+), 26 deletions(-) diff --git a/testdata/array-value.yaml b/testdata/array-value.yaml index 6c61d9f..90d5dc4 100644 --- a/testdata/array-value.yaml +++ b/testdata/array-value.yaml @@ -2,5 +2,5 @@ pipeline: filters: - name: record_modifier record: - - key1 value1 - - key2 value2 + - key1 value1 + - key2 value2 diff --git a/testdata/full.yaml b/testdata/full.yaml index a0e7ded..72869ab 100644 --- a/testdata/full.yaml +++ b/testdata/full.yaml @@ -25,8 +25,8 @@ pipeline: - name: record_modifier match: dummy record: - - key1 value1 - - key2 value2 + - key1 value1 + - key2 value2 outputs: - name: stdout match: '*' diff --git a/testdata/multiline-value.yaml b/testdata/multiline-value.yaml index c6bc448..821f96f 100644 --- a/testdata/multiline-value.yaml +++ b/testdata/multiline-value.yaml @@ -1,5 +1,5 @@ customs: - name: custom multiline: |- - FOO - BAR + FOO + BAR diff --git a/testdata/processors/processors.yaml b/testdata/processors/processors.yaml index e0a1da1..6e24108 100644 --- a/testdata/processors/processors.yaml +++ b/testdata/processors/processors.yaml @@ -8,28 +8,28 @@ pipeline: - name: dummy tag: test-tag processors: - logs: - - add: hostname monox - name: modify - - call: append_tag - code: | - function append_tag(tag, timestamp, record) - new_record = record - new_record["tag"] = tag - return 1, timestamp, new_record - end - name: lua + logs: + - add: hostname monox + name: modify + - call: append_tag + code: | + function append_tag(tag, timestamp, record) + new_record = record + new_record["tag"] = tag + return 1, timestamp, new_record + end + name: lua outputs: - name: stdout format: json_lines match: '*' processors: - logs: - - call: add_field - code: | - function add_field(tag, timestamp, record) - new_record = record - new_record["output"] = "new data" - return 1, timestamp, new_record - end - name: lua + logs: + - call: add_field + code: | + function add_field(tag, timestamp, record) + new_record = record + new_record["output"] = "new data" + return 1, timestamp, new_record + end + name: lua From bf163c8e273d29f1be6f3529fc67168c2eb8a142 Mon Sep 17 00:00:00 2001 From: Phillip Adair Stewart Whelan Date: Tue, 6 May 2025 13:23:42 -0400 Subject: [PATCH 5/7] testdata: change few remaining uses of double quotes to single quotes. Signed-off-by: Phillip Adair Stewart Whelan --- testdata/empty-string.yaml | 2 +- testdata/example-multiline-parser.yaml | 2 +- testdata/full.yaml | 4 ++-- testdata/processors/processors.yaml | 2 +- testdata/service.yaml | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/testdata/empty-string.yaml b/testdata/empty-string.yaml index d8d58a0..2ee973e 100644 --- a/testdata/empty-string.yaml +++ b/testdata/empty-string.yaml @@ -1,3 +1,3 @@ customs: - name: empty_string - value: "" + value: '' diff --git a/testdata/example-multiline-parser.yaml b/testdata/example-multiline-parser.yaml index 1e1b574..484af59 100644 --- a/testdata/example-multiline-parser.yaml +++ b/testdata/example-multiline-parser.yaml @@ -4,4 +4,4 @@ pipeline: Path: /var/log/containers/*.log Tag: containers multiline.parser: docker , cri - Skip_Long_Lines: "On" + Skip_Long_Lines: 'On' diff --git a/testdata/full.yaml b/testdata/full.yaml index 72869ab..4b8a016 100644 --- a/testdata/full.yaml +++ b/testdata/full.yaml @@ -3,9 +3,9 @@ env: BAZ: qux service: flush: 1 - daemon: "on" + daemon: 'on' log_level: error - http_server: "on" + http_server: 'on' customs: - name: calyptia api_key: secret diff --git a/testdata/processors/processors.yaml b/testdata/processors/processors.yaml index 6e24108..6ee802e 100644 --- a/testdata/processors/processors.yaml +++ b/testdata/processors/processors.yaml @@ -1,6 +1,6 @@ service: log_level: info - http_server: "on" + http_server: 'on' http_listen: 0.0.0.0 http_port: 2021 pipeline: diff --git a/testdata/service.yaml b/testdata/service.yaml index 0745a90..944f8e2 100644 --- a/testdata/service.yaml +++ b/testdata/service.yaml @@ -1,5 +1,5 @@ service: flush: 1 - daemon: "on" + daemon: 'on' log_level: error - http_server: "on" + http_server: 'on' From 99c3114154ac8a11c029158cf1b6cac175ef0d6d Mon Sep 17 00:00:00 2001 From: Phillip Adair Stewart Whelan Date: Tue, 6 May 2025 13:24:12 -0400 Subject: [PATCH 6/7] validator: allow uint64 as service property values. Signed-off-by: Phillip Adair Stewart Whelan --- validator.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/validator.go b/validator.go index 947f939..1f7e58c 100644 --- a/validator.go +++ b/validator.go @@ -35,11 +35,12 @@ func (c Config) ValidateWithSchema(schema Schema) error { switch property.Value.(type) { case bool: case int: + case uint64: case int64: case float64: case string: default: - return fmt.Errorf("invalid type (%T) for service setting: %s", + return fmt.Errorf("invalid type (%T) for service setting: %s", property.Value, property.Key) } From cfae57d8ac9e0e176777c7e3fc86f0de64b46275 Mon Sep 17 00:00:00 2001 From: Phillip Adair Stewart Whelan Date: Fri, 23 May 2025 17:57:51 -0400 Subject: [PATCH 7/7] fix: update error messages for goccy/yaml in the dump tests. Signed-off-by: Phillip Adair Stewart Whelan --- dump_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dump_test.go b/dump_test.go index 2ed1ac6..3e4e13f 100644 --- a/dump_test.go +++ b/dump_test.go @@ -21,7 +21,7 @@ func TestParseAsYAML(t *testing.T) { inputs: - name: dummy `)) - require.EqualError(t, err, "yaml: unmarshal errors:\n line 1: field pipelines not found in type fluentbitconfig.Config") + require.EqualError(t, err, "[1:1] unknown field \"pipelines\"\n> 1 | pipelines:\n ^\n 2 | inputs:\n 3 | - name: dummy") }) t.Run("unknown_section", func(t *testing.T) { @@ -29,7 +29,7 @@ func TestParseAsYAML(t *testing.T) { unknown: - name: dummy `)) - require.EqualError(t, err, "yaml: unmarshal errors:\n line 1: field unknown not found in type fluentbitconfig.Config") + require.EqualError(t, err, "[1:1] unknown field \"unknown\"\n> 1 | unknown:\n ^\n 2 | - name: dummy") }) t.Run("unknown_pipeline_plugins_kind", func(t *testing.T) { @@ -38,7 +38,7 @@ func TestParseAsYAML(t *testing.T) { foos: - name: bar `)) - require.EqualError(t, err, "yaml: unmarshal errors:\n line 2: field foos not found in type fluentbitconfig.Pipeline") + require.EqualError(t, err, "[2:5] unknown field \"foos\"\n 1 | pipeline:\n> 2 | foos:\n ^\n 3 | - name: bar") }) t.Run("empty", func(t *testing.T) {