diff --git a/Gopkg.lock b/Gopkg.lock index 82d0761..2891218 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -4,7 +4,11 @@ [[projects]] branch = "master" name = "code.cloudfoundry.org/commandrunner" - packages = [".","fake_command_runner","linux_command_runner"] + packages = [ + ".", + "fake_command_runner", + "linux_command_runner" + ] revision = "501fd662150bcea5a42a9b5c1c283209720788c3" [[projects]] @@ -44,9 +48,12 @@ [[projects]] name = "github.com/geofffranks/spruce" - packages = [".","log"] - revision = "9e4dce1aa5cfad08642cd064973b529ab0c2b8eb" - version = "v1.17.0" + packages = [ + ".", + "log" + ] + revision = "133dd34c71cd2dd46457e146d2e0e6e525da3ecc" + version = "v1.18.2" [[projects]] branch = "v2" @@ -62,12 +69,44 @@ [[projects]] name = "github.com/onsi/ginkgo" - packages = [".","config","internal/codelocation","internal/containernode","internal/failer","internal/leafnodes","internal/remote","internal/spec","internal/spec_iterator","internal/specrunner","internal/suite","internal/testingtproxy","internal/writer","reporters","reporters/stenographer","reporters/stenographer/support/go-colorable","reporters/stenographer/support/go-isatty","types"] + packages = [ + ".", + "config", + "internal/codelocation", + "internal/containernode", + "internal/failer", + "internal/leafnodes", + "internal/remote", + "internal/spec", + "internal/spec_iterator", + "internal/specrunner", + "internal/suite", + "internal/testingtproxy", + "internal/writer", + "reporters", + "reporters/stenographer", + "reporters/stenographer/support/go-colorable", + "reporters/stenographer/support/go-isatty", + "types" + ] revision = "11459a886d9cd66b319dac7ef1e917ee221372c9" [[projects]] name = "github.com/onsi/gomega" - packages = [".","format","internal/assertion","internal/asyncassertion","internal/oraclematcher","internal/testingtsupport","matchers","matchers/support/goraph/bipartitegraph","matchers/support/goraph/edge","matchers/support/goraph/node","matchers/support/goraph/util","types"] + packages = [ + ".", + "format", + "internal/assertion", + "internal/asyncassertion", + "internal/oraclematcher", + "internal/testingtsupport", + "matchers", + "matchers/support/goraph/bipartitegraph", + "matchers/support/goraph/edge", + "matchers/support/goraph/node", + "matchers/support/goraph/util", + "types" + ] revision = "dcabb60a477c2b6f456df65037cb6708210fbb02" [[projects]] @@ -82,7 +121,10 @@ [[projects]] name = "github.com/starkandwayne/goutils" - packages = ["ansi","tree"] + packages = [ + "ansi", + "tree" + ] revision = "2c96629058fe59038b1f090194d047ecf0194045" [[projects]] @@ -99,7 +141,11 @@ [[projects]] name = "golang.org/x/net" - packages = ["html","html/atom","html/charset"] + packages = [ + "html", + "html/atom", + "html/charset" + ] revision = "66aacef3dd8a676686c7ae3716979581e8b03c47" [[projects]] @@ -109,7 +155,25 @@ [[projects]] name = "golang.org/x/text" - packages = ["encoding","encoding/charmap","encoding/htmlindex","encoding/internal","encoding/internal/identifier","encoding/japanese","encoding/korean","encoding/simplifiedchinese","encoding/traditionalchinese","encoding/unicode","internal/gen","internal/tag","internal/utf8internal","language","runes","transform","unicode/cldr"] + packages = [ + "encoding", + "encoding/charmap", + "encoding/htmlindex", + "encoding/internal", + "encoding/internal/identifier", + "encoding/japanese", + "encoding/korean", + "encoding/simplifiedchinese", + "encoding/traditionalchinese", + "encoding/unicode", + "internal/gen", + "internal/tag", + "internal/utf8internal", + "language", + "runes", + "transform", + "unicode/cldr" + ] revision = "bd91bbf73e9a4a801adbfb97133c992678533126" [[projects]] diff --git a/README.md b/README.md index 1eff6a7..82a1969 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ If you have to handle rather complex YAML files (for BOSH or Concourse), you jus ### OS X ``` -$ wget -O /usr/local/bin/aviator https://github.com/JulzDiverse/aviator/releases/download/v0.20.0/aviator-darwin-amd64 && chmod +x /usr/local/bin/aviator +$ wget -O /usr/local/bin/aviator https://github.com/JulzDiverse/aviator/releases/download/v1.0.0/aviator-darwin-amd64 && chmod +x /usr/local/bin/aviator ``` **Via Homebrew** @@ -26,13 +26,13 @@ $ brew install aviator ### Linux ``` -$ wget -O /usr/bin/aviator https://github.com/JulzDiverse/aviator/releases/download/v0.20.0/aviator-linux-amd64 && chmod +x /usr/bin/aviator +$ wget -O /usr/bin/aviator https://github.com/JulzDiverse/aviator/releases/download/v1.0.0/aviator-linux-amd64 && chmod +x /usr/bin/aviator ``` ### Windows (NOT TESTED) ``` -https://github.com/JulzDiverse/aviator/releases/download/v0.20.0/aviator-win +https://github.com/JulzDiverse/aviator/releases/download/v1.0.0/aviator-win ``` ## Executors @@ -590,6 +590,29 @@ Note, that the generated `pipeline.yml` is used in the `fly` section as `config` _NOTE: You will need to fly login first, before executing `aviator`_ + +--- + +### CLI Options + +#### `--curly-braces` + +Some YAML based tools (like concourse in the past) are using `{{}}` sytnax. This is not YAML conform. Using the `--curly-braces` option you can allow this syntax. + +#### `--silent` + +This option will output no infromation to stdout. + +#### `--verbose` + +This option prints which files are excluded from a merge. + +#### `--var` + +You can provide variables to the aviator file. + +--- + # Development ``` diff --git a/cmd/aviator/flags.go b/cmd/aviator/flags.go index 14d3a57..2b20bfa 100644 --- a/cmd/aviator/flags.go +++ b/cmd/aviator/flags.go @@ -37,6 +37,10 @@ func getFlags() []cli.Flag { Name: "var", Usage: "provides a variable to an aviator file: [key=value]", }, + cli.BoolFlag{ + Name: "curly-braces, b", + Usage: "allow {{}} syntax in yaml files", + }, } return flags } diff --git a/cmd/aviator/main.go b/cmd/aviator/main.go index a63124f..5eb0f4b 100644 --- a/cmd/aviator/main.go +++ b/cmd/aviator/main.go @@ -18,9 +18,7 @@ func main() { cmd.Action = func(c *cli.Context) error { aviatorFile := c.String("file") if !verifyAviatorFileExists(aviatorFile) { - exitWithNoAviatorFile() - } else { vars := c.StringSlice("var") varsMap := varsToMap(vars) @@ -28,13 +26,19 @@ func main() { aviatorYml, err := ioutil.ReadFile(aviatorFile) exitWithError(err) - cockpit := cockpit.New() + cockpit := cockpit.New(c.Bool("curly-braces")) aviator, err := cockpit.NewAviator(aviatorYml, varsMap) handleError(err) err = aviator.ProcessSprucePlan(c.Bool("verbose"), c.Bool("silent")) exitWithError(err) + squash := aviator.AviatorYaml.Squash + if len(squash.Content) != 0 { + err = aviator.ProcessSquashPlan() + exitWithError(err) + } + fly := aviator.AviatorYaml.Fly if fly.Name != "" && fly.Target != "" && fly.Config != "" { err = aviator.ExecuteFly() diff --git a/cockpit/cockpit.go b/cockpit/cockpit.go index 057679a..bd57c9c 100644 --- a/cockpit/cockpit.go +++ b/cockpit/cockpit.go @@ -6,7 +6,9 @@ import ( "github.com/JulzDiverse/aviator" "github.com/JulzDiverse/aviator/evaluator" "github.com/JulzDiverse/aviator/executor" + "github.com/JulzDiverse/aviator/filemanager" "github.com/JulzDiverse/aviator/processor" + "github.com/JulzDiverse/aviator/squasher" "github.com/JulzDiverse/aviator/validator" "github.com/JulzDiverse/osenv" "github.com/pkg/errors" @@ -34,9 +36,9 @@ func Init( return &Cockpit{spruceProcessor, flyExecuter, validator} } -func New() *Cockpit { +func New(curlyBraces bool) *Cockpit { return &Cockpit{ - spruceProcessor: processor.New(), + spruceProcessor: processor.New(curlyBraces), validator: validator.New(), flyExecutor: executor.NewFlyExecutor(), } @@ -76,6 +78,40 @@ func (a *Aviator) ProcessSprucePlan(verbose bool, silent bool) error { return nil } +func (a *Aviator) ProcessSquashPlan() error { + var err error + var result []byte + + store := filemanager.Store(false) + fp := processor.FileProcessor{store} + + content := a.AviatorYaml.Squash.Content + for _, c := range content { + var squashed []byte + if len(c.Files) != 0 { + files := store.ReadFiles(c.Files) + squashed, err = squasher.Squash(files) + } else { + paths := fp.CollectFilesFromDir(c.Dir, "", []string{}) + files := store.ReadFiles(paths) + squashed, err = squasher.Squash(files) + } + + if err != nil { + return err + } + + result = append(result, squashed...) + } + + err = store.WriteFile(a.AviatorYaml.Squash.To, result) + if err != nil { + return err + } + + return nil +} + func (a *Aviator) ExecuteFly() error { err := a.cockpit.flyExecutor.Execute(a.AviatorYaml.Fly) if err != nil { diff --git a/filemanager/filemanager.go b/filemanager/filemanager.go index a87628f..ecfbd84 100644 --- a/filemanager/filemanager.go +++ b/filemanager/filemanager.go @@ -12,7 +12,8 @@ import ( ) type FileManager struct { - root *mingoak.Dir + CurlyBraces bool + root *mingoak.Dir } //var quoteRegexOld = `\{\{([-\_\.\/\w\p{L}\/]+)\}\}` @@ -21,9 +22,9 @@ var re = regexp.MustCompile("(" + quoteRegex + ")") var dere = regexp.MustCompile("['\"](" + quoteRegex + ")[\"']") var store *FileManager -func Store() *FileManager { +func Store(curlyBraces bool) *FileManager { if store == nil { - store = &FileManager{mingoak.MkRoot()} + store = &FileManager{curlyBraces, mingoak.MkRoot()} } return store } @@ -46,8 +47,20 @@ func (ds *FileManager) ReadFile(key string) ([]byte, bool) { return file, true } +func (ds *FileManager) ReadFiles(keys []string) [][]byte { + result := [][]byte{} + for _, k := range keys { + file, _ := ds.ReadFile(k) + result = append(result, file) + } + return result +} + func (ds *FileManager) WriteFile(key string, file []byte) error { - file = dequoteCurlyBraces(file) + if ds.CurlyBraces { + file = dequoteCurlyBraces(file) + } + if re.MatchString(key) { key = getKeyFromRegexp(key) //if _, err := ds.root.ReadFile(key); err == nil { diff --git a/filemanager/filemanager_test.go b/filemanager/filemanager_test.go index 57d9e08..2f16bd1 100644 --- a/filemanager/filemanager_test.go +++ b/filemanager/filemanager_test.go @@ -10,9 +10,14 @@ import ( var _ = Describe("Filemanager", func() { var store *FileManager + var allowCurlyBraces bool BeforeEach(func() { - store = Store() + allowCurlyBraces = true + }) + + JustBeforeEach(func() { + store = Store(allowCurlyBraces) }) Context("Write/ReadFile", func() { @@ -69,4 +74,18 @@ var _ = Describe("Filemanager", func() { //Expect(err).ToNot(HaveOccurred()) //}) //}) + + Context("When double curly braces are not allowed", func() { + BeforeEach(func() { + allowCurlyBraces = false + }) + + It("doesn't unquote curly braces on write", func() { + err := store.WriteFile("{{keyE}}", []byte("{{content E}}")) + Expect(err).ToNot(HaveOccurred()) + file, ok := store.ReadFile("{{keyE}}") + Expect(ok).To(Equal(true)) + Expect(string(file)).To(ContainSubstring("{{content E}}")) + }) + }) }) diff --git a/models.go b/models.go index 3a68342..fa628c4 100644 --- a/models.go +++ b/models.go @@ -5,6 +5,7 @@ import "os" type AviatorYaml struct { Spruce []Spruce `yaml:"spruce"` Fly Fly `yaml:"fly"` + Squash Squash `yaml:"squash"` } type Spruce struct { @@ -77,6 +78,16 @@ type PathVal struct { Value string `yaml:"value"` } +type Squash struct { + Content []SquashContent `yaml:"content"` + To string `yaml:"to"` +} + +type SquashContent struct { + Files []string `yaml:"files"` + Dir string `yaml:"dir"` +} + //go:generate counterfeiter . SpruceProcessor type SpruceProcessor interface { Process([]Spruce) error diff --git a/processor/processor.go b/processor/processor.go index 4ac6496..a2b4dbc 100644 --- a/processor/processor.go +++ b/processor/processor.go @@ -33,10 +33,10 @@ func NewTestProcessor(spruceClient aviator.SpruceClient, store aviator.FileStore } } -func New() *Processor { +func New(curlyBraces bool) *Processor { return &Processor{ - store: filemanager.Store(), - spruceClient: spruce.New(), + store: filemanager.Store(curlyBraces), + spruceClient: spruce.New(curlyBraces), modifier: modifier.New(), } } diff --git a/processor/processor_test.go b/processor/processor_test.go index 054a74c..b472c20 100644 --- a/processor/processor_test.go +++ b/processor/processor_test.go @@ -34,7 +34,7 @@ var _ = Describe("Processor", func() { To: "integration/tmp/result.yml", ToDir: "integration/tmp/", } - store = filemanager.Store() //new(fakes.FakeFileStore) + store = filemanager.Store(true) modifier = new(fakes.FakeModifier) }) diff --git a/processor/squash_processor.go b/processor/squash_processor.go new file mode 100644 index 0000000..5c32b71 --- /dev/null +++ b/processor/squash_processor.go @@ -0,0 +1,31 @@ +package processor + +import ( + "path/filepath" + "regexp" + + "github.com/JulzDiverse/aviator" +) + +type FileProcessor struct { + Store aviator.FileStore +} + +func (f *FileProcessor) CollectFilesFromDir(dir, regex string, ignore []string) []string { + result := []string{} + if dir != "" { + files, _ := f.Store.ReadDir(dir) + + for _, f := range files { + if except(ignore, f.Name()) { + continue + } + + matched, _ := regexp.MatchString(regex, f.Name()) + if !f.IsDir() && matched { + result = append(result, filepath.Join(resolveBraces(dir)+f.Name())) + } + } + } + return result +} diff --git a/spruce/spruce.go b/spruce/spruce.go index 89df4a9..323b5f3 100644 --- a/spruce/spruce.go +++ b/spruce/spruce.go @@ -14,21 +14,24 @@ import ( ) type SpruceClient struct { - store aviator.FileStore + CurlyBraces bool + store aviator.FileStore } var concourseRegex = `(\{\{|\+\+)([-\_\.\/\w\p{L}\/]+)(\}\}|\+\+)` var re = regexp.MustCompile("(" + concourseRegex + ")") var dere = regexp.MustCompile("['\"](" + concourseRegex + ")[\"']") -func New() *SpruceClient { +func New(curlyBraces bool) *SpruceClient { return &SpruceClient{ - filemanager.Store(), + curlyBraces, + filemanager.Store(curlyBraces), } } -func NewWithFileFilemanager(filemanager aviator.FileStore) *SpruceClient { +func NewWithFileFilemanager(filemanager aviator.FileStore, curlyBraces bool) *SpruceClient { return &SpruceClient{ + curlyBraces, filemanager, } } @@ -79,7 +82,11 @@ func (sc *SpruceClient) mergeAllDocs(root map[interface{}]interface{}, paths []s if !ok { return ansi.Errorf("@R{Error reading file from filesystem or internal datastore} @m{%s} \n", path) } - data = quoteConcourse(data) + + if sc.CurlyBraces { + data = quoteConcourse(data) + } + doc, err := parseYAML(data) if err != nil { if isArrayError(err) && goPatchEnabled { diff --git a/spruce/spruce_test.go b/spruce/spruce_test.go index a9c1a0b..aa476d0 100644 --- a/spruce/spruce_test.go +++ b/spruce/spruce_test.go @@ -15,7 +15,7 @@ var _ = Describe("Spruce", func() { BeforeEach(func() { spruce = NewWithFileFilemanager( - filemanager.Store(), + filemanager.Store(true), true, ) }) diff --git a/squasher/check b/squasher/check new file mode 100644 index 0000000..89b9edf --- /dev/null +++ b/squasher/check @@ -0,0 +1,4 @@ +--- +i_am_yaml: 1 +--- +i_am_yaml: 2 diff --git a/squasher/squash.go b/squasher/squash.go new file mode 100644 index 0000000..d6ceb42 --- /dev/null +++ b/squasher/squash.go @@ -0,0 +1,28 @@ +package squasher + +import ( + "bufio" + "bytes" + "errors" + "fmt" +) + +func Squash(files [][]byte) ([]byte, error) { + result := []byte{} + if len(files) <= 1 { + return nil, errors.New("zero or one file provided to squash") + } + + for _, file := range files { + bytesReader := bytes.NewReader(file) + bufReader := bufio.NewReader(bytesReader) + firstLine, _, _ := bufReader.ReadLine() + if string(firstLine) != "---" && string(firstLine) != "" { + file = append([]byte(fmt.Sprintf("---\n")), file...) + } + + file = append(file, []byte(fmt.Sprintf("\n"))...) + result = append(result, file...) + } + return result, nil +} diff --git a/squasher/squash_test.go b/squasher/squash_test.go new file mode 100644 index 0000000..a9f32df --- /dev/null +++ b/squasher/squash_test.go @@ -0,0 +1,83 @@ +package squasher_test + +import ( + "io/ioutil" + "strings" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + . "github.com/JulzDiverse/aviator/squasher" +) + +var _ = Describe("Squash", func() { + + var ( + yamls [][]byte + squashed []byte + err error + ) + + JustBeforeEach(func() { + squashed, err = Squash(yamls) + }) + + Context("When providing multiple yaml files", func() { + BeforeEach(func() { + yaml1 := `--- +i_am_yaml: 1` + + yaml2 := `--- +i_am_yaml: 2` + + yamls = [][]byte{[]byte(yaml1), []byte(yaml2)} + }) + + It("should not fail", func() { + Expect(err).ToNot(HaveOccurred()) + }) + + It("squashes it to a single yaml file", func() { + expected := `--- +i_am_yaml: 1 +--- +i_am_yaml: 2` + + Expect(strings.Trim(string(squashed), "\n")).To(Equal(strings.Trim(string(expected), "\n"))) + + ioutil.WriteFile("check", squashed, 0644) + }) + }) + + Context("When providing less than two yaml files", func() { + Context("When providing a single yaml file", func() { + BeforeEach(func() { + yamls = [][]byte{{}} + squashed, err = Squash(yamls) + }) + + It("should fail", func() { + Expect(err).To(HaveOccurred()) + }) + + It("should not try to squash anything", func() { + Expect(squashed).To(BeNil()) + }) + }) + + Context("When providing no yaml file", func() { + BeforeEach(func() { + yamls = [][]byte{} + squashed, err = Squash(yamls) + }) + + It("should fail", func() { + Expect(err).To(HaveOccurred()) + }) + + It("should not try to squash anything", func() { + Expect(squashed).To(BeNil()) + }) + }) + }) +}) diff --git a/squasher/squasher_suite_test.go b/squasher/squasher_suite_test.go new file mode 100644 index 0000000..8984581 --- /dev/null +++ b/squasher/squasher_suite_test.go @@ -0,0 +1,13 @@ +package squasher_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestSquasher(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Squasher Suite") +} diff --git a/vendor/github.com/geofffranks/spruce/README.md b/vendor/github.com/geofffranks/spruce/README.md index 7361f9a..b860067 100644 --- a/vendor/github.com/geofffranks/spruce/README.md +++ b/vendor/github.com/geofffranks/spruce/README.md @@ -100,7 +100,7 @@ helpful subcommands: semantically. This is more than a simple diff tool, as it examines the functional differences, rather than just textual (e.g. key-ordering differences would be ignored) -`spruce json` - Allows you to convert a YAML docuemnt into JSON, for consumption by something +`spruce json` - Allows you to convert a YAML document into JSON, for consumption by something that requires a JSON input. `spruce merge` will handle both YAML + JSON documents, but produce only YAML output. diff --git a/vendor/github.com/geofffranks/spruce/cmd/spruce/main.go b/vendor/github.com/geofffranks/spruce/cmd/spruce/main.go index 658f4d9..90b5553 100644 --- a/vendor/github.com/geofffranks/spruce/cmd/spruce/main.go +++ b/vendor/github.com/geofffranks/spruce/cmd/spruce/main.go @@ -13,7 +13,6 @@ import ( . "github.com/geofffranks/spruce" . "github.com/geofffranks/spruce/log" - "regexp" "strings" // Use geofffranks forks to persist the fix in https://github.com/go-yaml/yaml/pull/133/commits @@ -46,13 +45,16 @@ var usage = func() { exit(1) } -var handleConcourseQuoting bool - func envFlag(varname string) bool { val := os.Getenv(varname) return val != "" && strings.ToLower(val) != "false" && val != "0" } +type jsonOpts struct { + Strict bool `goptions:"--strict, description='Refuse to convert non-string keys to strings'"` + Files goptions.Remainder `goptions:"description='Files to convert to JSON'"` +} + type mergeOpts struct { SkipEval bool `goptions:"--skip-eval, description='Do not evaluate spruce logic after merging docs'"` Prune []string `goptions:"--prune, description='Specify keys to prune from final output (may be specified more than once)'"` @@ -64,15 +66,12 @@ type mergeOpts struct { func main() { var options struct { - Debug bool `goptions:"-D, --debug, description='Enable debugging'"` - Trace bool `goptions:"-T, --trace, description='Enable trace mode debugging (very verbose)'"` - Version bool `goptions:"-v, --version, description='Display version information'"` - Concourse bool `goptions:"--concourse, description='Pre/Post-process YAML for Concourse CI (handles {{ }} quoting)'"` - Action goptions.Verbs - Merge mergeOpts `goptions:"merge"` - JSON struct { - Files goptions.Remainder `goptions:"description='Files to convert to JSON'"` - } `goptions:"json"` + Debug bool `goptions:"-D, --debug, description='Enable debugging'"` + Trace bool `goptions:"-T, --trace, description='Enable trace mode debugging (very verbose)'"` + Version bool `goptions:"-v, --version, description='Display version information'"` + Action goptions.Verbs + Merge mergeOpts `goptions:"merge"` + JSON jsonOpts `goptions:"json"` Diff struct { Files goptions.Remainder `goptions:"description='Show the semantic differences between two YAML files'"` } `goptions:"diff"` @@ -92,8 +91,6 @@ func main() { DebugOn = true } - handleConcourseQuoting = options.Concourse - if options.Version { printfStdOut("%s - Version %s\n", os.Args[0], Version) exit(0) @@ -120,14 +117,7 @@ func main() { return } - var output string - if handleConcourseQuoting { - PrintfStdErr(ansi.Sprintf("@Y{--concourse is deprecated. Consider using built-in spruce operators when merging Concourse YAML files}\n")) - output = dequoteConcourse(merged) - } else { - output = string(merged) - } - printfStdOut("%s\n", output) + printfStdOut("%s\n", string(merged)) case "vaultinfo": VaultRefs = map[string][]string{} @@ -144,7 +134,7 @@ func main() { printfStdOut("%s\n", formatVaultRefs()) case "json": if len(options.JSON.Files) > 0 { - jsons, err := JSONifyFiles(options.JSON.Files) + jsons, err := JSONifyFiles(options.JSON.Files, options.JSON.Strict) if err != nil { PrintfStdErr("%s\n", err) exit(2) @@ -154,7 +144,7 @@ func main() { printfStdOut("%s\n", output) } } else { - output, err := JSONifyIO(os.Stdin) + output, err := JSONifyIO(os.Stdin, options.JSON.Strict) if err != nil { PrintfStdErr("%s\n", err) exit(2) @@ -300,10 +290,6 @@ func mergeAllDocs(root map[interface{}]interface{}, paths []string, fallbackAppe } } - if handleConcourseQuoting { - data = quoteConcourse(data) - } - doc, err := parseYAML(data) if err != nil { if isArrayError(err) && goPatchEnabled { @@ -334,18 +320,6 @@ func mergeAllDocs(root map[interface{}]interface{}, paths []string, fallbackAppe return m.Error() } -var concourseRegex = `\{\{([-\w\p{L}]+)\}\}` - -func quoteConcourse(input []byte) []byte { - re := regexp.MustCompile("(" + concourseRegex + ")") - return re.ReplaceAll(input, []byte("\"$1\"")) -} - -func dequoteConcourse(input []byte) string { - re := regexp.MustCompile("['\"](" + concourseRegex + ")[\"']") - return re.ReplaceAllString(string(input), "$1") -} - func diffFiles(paths []string) (string, bool, error) { if len(paths) != 2 { return "", false, ansi.Errorf("incorrect number of files given to diffFiles(); please file a bug report") diff --git a/vendor/github.com/geofffranks/spruce/cmd/spruce/main_test.go b/vendor/github.com/geofffranks/spruce/cmd/spruce/main_test.go index afab425..3dc37d0 100644 --- a/vendor/github.com/geofffranks/spruce/cmd/spruce/main_test.go +++ b/vendor/github.com/geofffranks/spruce/cmd/spruce/main_test.go @@ -274,20 +274,6 @@ map: `) So(stderr, ShouldEqual, "") }) - Convey("Should not fail when handling concourse-style yaml and --concourse", func() { - os.Args = []string{"spruce", "--concourse", "merge", "../../assets/concourse/first.yml", "../../assets/concourse/second.yml"} - stdout = "" - stderr = "" - main() - So(stdout, ShouldEqual, `jobs: -- curlies: {{my-variable_123}} - name: thing1 -- curlies: {{more}} - name: thing2 - -`) - So(stderr, ShouldEqual, "--concourse is deprecated. Consider using built-in spruce operators when merging Concourse YAML files\n") - }) Convey("Should not evaluate spruce logic when --no-eval", func() { os.Args = []string{"spruce", "merge", "--skip-eval", "../../assets/no-eval/first.yml", "../../assets/no-eval/second.yml"} @@ -1332,6 +1318,100 @@ z: } }) + Convey("Sort test cases", func() { + Convey("sort operator functionality", func() { + os.Args = []string{"spruce", "merge", "../../assets/sort/base.yml", "../../assets/sort/op.yml"} + stdout = "" + stderr = "" + main() + So(stderr, ShouldEqual, "") + So(stdout, ShouldEqual, `float_list: +- 1.42 +- 2.42 +- 3.42 +- 4.42 +- 5.42 +- 6.42 +- 7.42 +- 8.42 +- 9.42 +foobar_list: +- foobar: item-6 +- foobar: item-7 +- foobar: item-8 +- foobar: item-9 +- foobar: item-g +- foobar: item-h +- foobar: item-i +- foobar: item-j +- foobar: item-k +- foobar: item-l +- foobar: item-m +int_list: +- 1 +- 2 +- 3 +- 4 +- 5 +- 6 +- 7 +- 8 +- 9 +key_list: +- key: item-1 +- key: item-2 +- key: item-3 +- key: item-4 +- key: item-a +- key: item-b +- key: item-c +- key: item-d +- key: item-e +- key: item-f +- key: item-g +- key: item-h +- key: item-i +name_list: +- name: item-1 +- name: item-2 +- name: item-3 +- name: item-4 +- name: item-5 +- name: item-6 +- name: item-7 +- name: item-8 +- name: item-9 +- name: item-a +- name: item-b +- name: item-c +- name: item-d +- name: item-e +- name: item-f +- name: item-g +- name: item-h +- name: item-i +- name: item-j +- name: item-k +- name: item-l +- name: item-m +- name: item-n +- name: item-o +- name: item-p +- name: item-q +- name: item-r +- name: item-s +- name: item-t +- name: item-u +- name: item-v +- name: item-w +- name: item-x +- name: item-y +- name: item-z + +`) + }) + }) + Convey("Cherry picking test cases", func() { Convey("Cherry pick just one root level path", func() { os.Args = []string{"spruce", "merge", "--cherry-pick", "properties", "../../assets/cherry-pick/fileA.yml", "../../assets/cherry-pick/fileB.yml"} @@ -1824,6 +1904,7 @@ warning: Falling back to inline merge strategy `) So(stdout, ShouldEqual, "") }) + Convey("Issue #172 - error instead of panic if merge on key was specifically requested but target key has map value", func() { os.Args = []string{"spruce", "merge", "../../assets/issue-172/explicitmergeonkey1.yml"} stdout = "" @@ -1900,6 +1981,73 @@ meta: `) }) + + Convey("Issue #267 - specifying an explicit merge operator must behave in the same way as relying on the default implicit merge operation", func() { + Convey("Option 1 - standard use-case: no explicit merge, named-entry list identifier key is the default called 'name'", func() { + os.Args = []string{"spruce", "merge", "../../assets/issue-267/option1-fileA.yml", "../../assets/issue-267/option1-fileB.yml"} + stdout = "" + stderr = "" + + main() + So(stderr, ShouldEqual, "") + So(stdout, ShouldEqual, `serverFiles: + prometheus.yml: + scrape_configs: + - name: one + - name: two + +`) + }) + + Convey("Option 2 - academic version of the option 1: same set-up, but with explicit usage of the merge operator", func() { + os.Args = []string{"spruce", "merge", "../../assets/issue-267/option2-fileA.yml", "../../assets/issue-267/option2-fileB.yml"} + stdout = "" + stderr = "" + + main() + So(stderr, ShouldEqual, "") + So(stdout, ShouldEqual, `serverFiles: + prometheus.yml: + scrape_configs: + - name: one + - name: two + +`) + }) + + Convey("Option 3 - even more academic version of the option 1: same set-up, but with explicit usage of the merge operator and specification of the default identifier key called 'name'", func() { + os.Args = []string{"spruce", "merge", "../../assets/issue-267/option3-fileA.yml", "../../assets/issue-267/option3-fileB.yml"} + stdout = "" + stderr = "" + + main() + So(stderr, ShouldEqual, "") + So(stdout, ShouldEqual, `serverFiles: + prometheus.yml: + scrape_configs: + - name: one + - name: two + +`) + }) + + Convey("Option 4 - actual real world use case, where the identifier key is call 'job_name' and therefore explicit merge on key is required", func() { + os.Args = []string{"spruce", "merge", "../../assets/issue-267/option4-fileA.yml", "../../assets/issue-267/option4-fileB.yml"} + stdout = "" + stderr = "" + + main() + So(stderr, ShouldEqual, "") + So(stdout, ShouldEqual, `serverFiles: + prometheus.yml: + scrape_configs: + - job_name: one + - job_name: two + +`) + }) + }) + Convey("Support go-patch files", func() { Convey("go-patch can modify yaml files in the merge phase, and insert spruce operators as required", func() { os.Args = []string{"spruce", "merge", "--go-patch", "../../assets/go-patch/base.yml", "../../assets/go-patch/patch.yml", "../../assets/go-patch/toMerge.yml"} @@ -1974,7 +2122,6 @@ spruce_array_grab: os.Setenv("DEFAULT_ARRAY_MERGE_KEY", "") }) }) - } func TestDebug(t *testing.T) { @@ -2037,39 +2184,6 @@ func TestDebug(t *testing.T) { }) } -func TestQuoteConcourse(t *testing.T) { - Convey("quoteConcourse()", t, func() { - Convey("Correctly double-quotes incoming {{\\S}} patterns", func() { - Convey("adds quotes", func() { - input := []byte("name: {{var-_1able}}") - So(string(quoteConcourse(input)), ShouldEqual, "name: \"{{var-_1able}}\"") - }) - }) - Convey("doesn't affect regularly quoted things", func() { - input := []byte("name: \"my value\"") - So(string(quoteConcourse(input)), ShouldEqual, "name: \"my value\"") - }) - }) -} -func TestDequoteConcourse(t *testing.T) { - Convey("dequoteConcourse()", t, func() { - Convey("Correctly removes quotes from incoming {{\\S}} patterns", func() { - Convey("with single quotes", func() { - input := []byte("name: '{{var-_1able}}'") - So(dequoteConcourse(input), ShouldEqual, "name: {{var-_1able}}") - }) - Convey("with double quotes", func() { - input := []byte("name: \"{{var-_1able}}\"") - So(dequoteConcourse(input), ShouldEqual, "name: {{var-_1able}}") - }) - }) - Convey("doesn't affect regularly quoted things", func() { - input := []byte("name: \"my value\"") - So(dequoteConcourse(input), ShouldEqual, "name: \"my value\"") - }) - }) -} - func TestExamples(t *testing.T) { var stdout string printfStdOut = func(format string, args ...interface{}) { diff --git a/vendor/github.com/geofffranks/spruce/doc/merging.md b/vendor/github.com/geofffranks/spruce/doc/merging.md index 0041796..10ca7b4 100644 --- a/vendor/github.com/geofffranks/spruce/doc/merging.md +++ b/vendor/github.com/geofffranks/spruce/doc/merging.md @@ -66,7 +66,7 @@ and multiple phases have been introduced into `spruce` to handle the various tas ## What about arrays? -Merging arrays together is slightly more complicated than merging arbitray-key-values, +Merging arrays together is slightly more complicated than merging arbitrary-key-values, because order matters. To aid in this, `spruce` has specific **array operators** that are used to tell it how to perform array merges: diff --git a/vendor/github.com/geofffranks/spruce/doc/operators.md b/vendor/github.com/geofffranks/spruce/doc/operators.md index fb1cde8..5fc54cc 100644 --- a/vendor/github.com/geofffranks/spruce/doc/operators.md +++ b/vendor/github.com/geofffranks/spruce/doc/operators.md @@ -22,7 +22,7 @@ see the [array merging documentation][array-merging]: - `(( append ))` - Adds the data to the end of the corresponding array in the root document. - `(( prepend ))` - Inserts the data at the beginning of the corresponding array in the root document. - `(( insert ))` - Inserts the data before or after a specified index, or object. -- `(( merge ))` - Merges the data on top of an existing array based on a common key. This +- `(( merge ))` - Merges the data on top of an existing array based on a common key. This requires each element to be an object, all with the common key used for merging. - `(( inline ))` - Merges the data ontop of an existing array, based on the indices of the array. @@ -68,7 +68,7 @@ Usage: `(( concat LITERAL|REFERENCE ... ))` The `(( concat ))` operator has a role that may shock you. It concatenates values together into a string. You can pass it any number of arguments, literal or reference, as long as the reference -is not an array/map. +is not an array/map. [Example][concat-example] @@ -211,6 +211,26 @@ make use `(( prune ))`s to clear out the bloated data.. [Example][prune-example] +## (( sort )) + +Usage: `(( sort [by KEY] ))` + +This operator enables sorting simple lists like lists of strings, or +numbers as well as lists of maps that follow the known contract of containing an +identifying entry each, like `name`, `key`, or `id`. As always, `name` is the +default for named-entry lists if no sort key is specified. The `(( sort ))` +operator works similar to the `(( prune ))` operator as more of an annotation +than an actual operator that immediately performs an action: The path at which +the sort operator is used will be marked for evaluation in the post-processing +phase. That means the sorting will only take place once after all files are +merged. + +The `(( sort ))` operator will fail in case of: +- lists that do not contain strings, numbers or maps (for example lists of lists) +- inhomogeneous types (mixing strings and numbers) +- named-entry maps that do not have a single identifying entry + + ## (( static_ips )) Usage: `(( static_ips INTEGER ... ))` diff --git a/vendor/github.com/geofffranks/spruce/evaluator.go b/vendor/github.com/geofffranks/spruce/evaluator.go index 2e19b0b..1945bd1 100644 --- a/vendor/github.com/geofffranks/spruce/evaluator.go +++ b/vendor/github.com/geofffranks/spruce/evaluator.go @@ -258,7 +258,7 @@ func (ev *Evaluator) DataFlow(phase OperatorPhase) ([]*Opcall, error) { } sort.Strings(sortedKeys) - // find all nodes in g that are free (no futher dependencies) + // find all nodes in g that are free (no further dependencies) freeNodes := func(g [][]*Opcall) []*Opcall { l := []*Opcall{} for _, k := range sortedKeys { @@ -388,6 +388,50 @@ func (ev *Evaluator) Prune(paths []string) error { return nil } +// SortPaths sorts all paths (keys in map) using the provided sort-key (respective value) +func (ev *Evaluator) SortPaths(pathKeyMap map[string]string) error { + DEBUG("sorting %d paths in the final YAML structure", len(pathKeyMap)) + for path, sortBy := range pathKeyMap { + DEBUG(" sorting path %s (sort-key %s)", path, sortBy) + + cursor, err := tree.ParseCursor(path) + if err != nil { + return err + } + + value, err := cursor.Resolve(ev.Tree) + if err != nil { + return err + } + + switch value.(type) { + case []interface{}: + // no-op, that's what we want ... + + case map[interface{}]interface{}: + return tree.TypeMismatchError{ + Path: []string{path}, + Wanted: "a list", + Got: "a map", + } + + default: + return tree.TypeMismatchError{ + Path: []string{path}, + Wanted: "a list", + Got: "a scalar", + } + } + + if err := sortList(path, value.([]interface{}), sortBy); err != nil { + return err + } + } + + DEBUG("") + return nil +} + // Cherry-pick ... func (ev *Evaluator) CherryPick(paths []string) error { DEBUG("cherry-picking %d paths from the final YAML structure", len(paths)) @@ -645,6 +689,10 @@ func (ev *Evaluator) Run(prune []string, picks []string) error { errors.Append(ev.Prune(keysToPrune)) keysToPrune = nil + // post-processing: sorting + errors.Append(ev.SortPaths(pathsToSort)) + pathsToSort = map[string]string{} + // post-processing: cherry-pick errors.Append(ev.CherryPick(picks)) diff --git a/vendor/github.com/geofffranks/spruce/json.go b/vendor/github.com/geofffranks/spruce/json.go index 8eaea41..dc5c0a3 100644 --- a/vendor/github.com/geofffranks/spruce/json.go +++ b/vendor/github.com/geofffranks/spruce/json.go @@ -12,7 +12,7 @@ import ( . "github.com/geofffranks/spruce/log" ) -func jsonifyData(data []byte) (string, error) { +func jsonifyData(data []byte, strict bool) (string, error) { y, err := simpleyaml.NewYaml(data) if err != nil { return "", err @@ -23,7 +23,12 @@ func jsonifyData(data []byte) (string, error) { return "", ansi.Errorf("@R{Root of YAML document is not a hash/map}: %s\n", err.Error()) } - b, err := json.Marshal(deinterface(doc)) + doc_, err := deinterface(doc, strict) + if err != nil { + return "", err + } + + b, err := json.Marshal(doc_) if err != nil { return "", err } @@ -31,15 +36,15 @@ func jsonifyData(data []byte) (string, error) { return string(b), nil } -func JSONifyIO(in io.Reader) (string, error) { +func JSONifyIO(in io.Reader, strict bool) (string, error) { data, err := ioutil.ReadAll(in) if err != nil { return "", ansi.Errorf("@R{Error reading input}: %s", err) } - return jsonifyData(data) + return jsonifyData(data, strict) } -func JSONifyFiles(paths []string) ([]string, error) { +func JSONifyFiles(paths []string, strict bool) ([]string, error) { l := make([]string, len(paths)) for i, path := range paths { @@ -49,7 +54,7 @@ func JSONifyFiles(paths []string) ([]string, error) { return nil, ansi.Errorf("@R{Error reading file} @m{%s}: %s", path, err) } - if l[i], err = jsonifyData(data); err != nil { + if l[i], err = jsonifyData(data, strict); err != nil { return nil, ansi.Errorf("%s: %s", path, err) } } @@ -57,29 +62,62 @@ func JSONifyFiles(paths []string) ([]string, error) { return l, nil } -func deinterface(o interface{}) interface{} { +func deinterface(o interface{}, strict bool) (interface{}, error) { switch o.(type) { case map[interface{}]interface{}: - return deinterfaceMap(o.(map[interface{}]interface{})) + return deinterfaceMap(o.(map[interface{}]interface{}), strict) case []interface{}: - return deinterfaceList(o.([]interface{})) + return deinterfaceList(o.([]interface{}), strict) default: - return o + return o, nil } } -func deinterfaceMap(o map[interface{}]interface{}) map[string]interface{} { +func addKeyToMap(m map[string]interface{}, k interface{}, v interface{}, strict bool) (error) { + vs := fmt.Sprintf("%v", k) + _, exists := m[vs] + if exists { + NewWarningError(eContextAll, "@Y{Duplicate key detected: %s}", vs).Warn() + return nil + } + dv, err := deinterface(v, strict) + if err != nil { + return err + } + m[vs] = dv + return nil +} + +func deinterfaceMap(o map[interface{}]interface{}, strict bool) (map[string]interface{}, error) { m := map[string]interface{}{} for k, v := range o { - m[fmt.Sprintf("%v", k)] = deinterface(v) + + switch k.(type) { + case string: + err := addKeyToMap(m, k, v, strict) + if err != nil { + return nil, err + } + default: + if (strict) { + return nil, fmt.Errorf("Non-string keys found during strict JSON conversion") + } else { + addKeyToMap(m, k, v, strict) + } + } + } - return m + return m, nil } -func deinterfaceList(o []interface{}) []interface{} { +func deinterfaceList(o []interface{}, strict bool) ([]interface{}, error) { l := make([]interface{}, len(o)) for i, v := range o { - l[i] = deinterface(v) + v_, err := deinterface(v, strict) + if err != nil { + return nil, err + } + l[i] = v_ } - return l + return l, nil } diff --git a/vendor/github.com/geofffranks/spruce/merge.go b/vendor/github.com/geofffranks/spruce/merge.go index bc74493..34b957c 100644 --- a/vendor/github.com/geofffranks/spruce/merge.go +++ b/vendor/github.com/geofffranks/spruce/merge.go @@ -13,6 +13,17 @@ import ( . "github.com/geofffranks/spruce/log" ) +type listOp int + +const ( + listOpMergeDefault listOp = iota + listOpMergeOnKey + listOpMergeInline + listOpReplace + listOpInsert + listOpDelete +) + // Merger ... type Merger struct { AppendByDefault bool @@ -21,18 +32,20 @@ type Merger struct { depth int } -// Array operation helper structure +// ModificationDefinition encapsulates the details of an array modification: +// (1) the type of modification, e.g. insert, delete, replace +// (2) an optional guide to the specific part of the array to be modified, +// for example the index at which an insertion should be done +// (3) an optional list of entries to be added or merged into the array type ModificationDefinition struct { - index int - - key string - name string - - delete bool - defaultMerge bool // True if modification represents the default merge behavior + listOp listOp + index int + key string + name string relative string - list []interface{} + + list []interface{} } // Merge ... @@ -53,6 +66,16 @@ func (m *Merger) Error() error { return nil } +func getDefaultIdentifierKey() string { + // Use environment variable override, if set + if os.Getenv("DEFAULT_ARRAY_MERGE_KEY") != "" { + return os.Getenv("DEFAULT_ARRAY_MERGE_KEY") + } + + // the built-in default: name + return "name" +} + func deepCopy(orig interface{}) interface{} { switch orig.(type) { case map[interface{}]interface{}: @@ -99,20 +122,41 @@ func (m *Merger) mergeMap(orig map[interface{}]interface{}, n map[interface{}]in } func (m *Merger) mergeObj(orig interface{}, n interface{}, node string) interface{} { - // regular expression to search for prune operator to make its special behavior possible + // regular expression to search for prune and sort operator to make their + // special behavior possible pruneRx := regexp.MustCompile(`^\s*\Q((\E\s*prune\s*\Q))\E`) - - // prune special behavior I/II: if the value is replaced during processing (overwritten), the path will be removed at the end of the processing anyway - if origString, ok := orig.(string); ok && pruneRx.MatchString(origString) { + sortRx := regexp.MustCompile(`^\s*\Q((\E\s*sort(?:\s+by\s+(.*?))?\s*\Q))\E$`) + + // prune/sort operator special behavior I: + // operator is defined in the original object and will now be overwritten by + // the new value. Therefore, remember that the operator was here at that path + // + // prune/sort operator special behavior II: + // operator is defined in the new object and would therefore overwrite the + // original content. In this case, keep the original content as it is and mark + // that the operator occurred at this path + // + // common requirement is that both original and new object values are strings + origString, origOk := orig.(string) + newString, newOk := n.(string) + switch { + case origOk && pruneRx.MatchString(origString): DEBUG("%s: a (( prune )) operator is about to be replaced, check if its path needs to be saved", node) addToPruneListIfNecessary(strings.Replace(node, "$.", "", -1)) - } - // prune special behavior II/II: if a new prune operator would overwrite something, this will be omitted but the path saved for later pruning - if nString, ok := n.(string); ok && pruneRx.MatchString(nString) && orig != nil { + case newOk && pruneRx.MatchString(newString) && orig != nil: DEBUG("%s: a (( prune )) operator is about to replace existing content, check if its path needs to be saved", node) addToPruneListIfNecessary(strings.Replace(node, "$.", "", -1)) return orig + + case origOk && sortRx.MatchString(origString): + DEBUG("%s: a (( sort )) operator is about to be replaced, check if its path needs to be saved", node) + addToSortListIfNecessary(origString, strings.Replace(node, "$.", "", -1)) + + case newOk && sortRx.MatchString(newString) && orig != nil: + DEBUG("%s: a (( sort )) operator is about to replace existing content, check if its path needs to be saved", node) + addToSortListIfNecessary(newString, strings.Replace(node, "$.", "", -1)) + return orig } switch t := n.(type) { @@ -160,64 +204,74 @@ func (m *Merger) mergeObj(orig interface{}, n interface{}, node string) interfac } func (m *Merger) mergeArray(orig []interface{}, n []interface{}, node string) []interface{} { + modificationDefinitions := getArrayModifications(n, isSimpleList(orig)) + DEBUG("%s: performing %d modification operations against list", node, len(modificationDefinitions)) - if shouldInlineMergeArray(n) { - DEBUG("%s: performing explicit inline array merge", node) - return m.mergeArrayInline(orig, n[1:], node) + // Create a copy of orig for the (multiple) modifications that are about to happen + result := make([]interface{}, len(orig)) + copy(result, orig) - } else if shouldReplaceArray(n) { - DEBUG("%s: replacing with new data", node) - return n[1:] + // Process the modifications definitions that were found in the new list + for i, modificationDefinition := range modificationDefinitions { + DEBUG(" #%d %#v", i, modificationDefinition) - } else if should, key := shouldKeyMergeArray(n); should { - DEBUG("%s: performing key-based array merge, using key '%s'", node, key) + // insert/delete operations will use a list index later in this loop block + var idx int - if err := canKeyMergeArray("new", n[1:], node, key); err != nil { - m.Errors.Append(err) - return nil - } - if err := canKeyMergeArray("original", orig, node, key); err != nil { - m.Errors.Append(err) - return nil + // Special tag for default behavior. Cannot be invoked explicitly by users + if modificationDefinition.listOp == listOpMergeDefault { + result = m.mergeArrayDefault(orig, modificationDefinitions[0].list, node) + continue } - return m.mergeArrayByKey(orig, n[1:], node, key) - - } + // Perform a merge on key list modification (merge in new list on original) + if modificationDefinition.listOp == listOpMergeOnKey { + key := modificationDefinition.key + if key == "" { + key = getDefaultIdentifierKey() + } - isSimpleList := isSimpleList(orig) - modificationDefinitions := getArrayModifications(n, isSimpleList) - DEBUG("%s: performing %d modification operations against list", node, len(modificationDefinitions)) - DEBUG("%+v", modificationDefinitions) + if err := canKeyMergeArray("new", modificationDefinition.list, node, key); err != nil { + m.Errors.Append(err) + return nil + } + if err := canKeyMergeArray("original", orig, node, key); err != nil { + m.Errors.Append(err) + return nil + } - // Create a copy of orig for the (multiple) modifications that are about to happen - result := make([]interface{}, len(orig)) - copy(result, orig) + result = m.mergeArrayByKey(result, modificationDefinition.list, node, key) + continue + } - var idx int + // Perform a merge inline list modification + if modificationDefinition.listOp == listOpMergeInline { + result = m.mergeArrayInline(result, modificationDefinition.list, node) + continue + } - // Process the modifications definitions that were found in the new list - for i := range modificationDefinitions { - //Special tag for default behavior. Cannot be invoked explicitly by users - if modificationDefinitions[i].defaultMerge { - result = m.mergeArrayDefault(orig, modificationDefinitions[0].list, node) + // Perform a list replacement modification + if modificationDefinition.listOp == listOpReplace { + result = make([]interface{}, len(modificationDefinition.list)) + copy(result, modificationDefinition.list) continue } - if modificationDefinitions[i].key == "" && modificationDefinitions[i].name == "" { // Index comes directly from operation definition - idx = modificationDefinitions[i].index + // Perform insert, delete, append, prepend operation + if modificationDefinition.key == "" && modificationDefinition.name == "" { // Index comes directly from operation definition + idx = modificationDefinition.index // Replace the -1 marker with the actual 'end' index of the array if idx == -1 { idx = len(result) } - } else if modificationDefinitions[i].key == "" && modificationDefinitions[i].name != "" { - name := modificationDefinitions[i].name - delete := modificationDefinitions[i].delete + } else if modificationDefinition.key == "" && modificationDefinition.name != "" { + name := modificationDefinition.name + delete := modificationDefinition.listOp == listOpDelete if delete { // Sanity check for delete operation, ensure no orphan entries follow the operator definition - if len(modificationDefinitions[i].list) > 0 { + if len(modificationDefinition.list) > 0 { m.Errors.Append(ansi.Errorf("@m{%s}: @R{item in array directly after} @c{(( delete \"%s\" ))} @r{must be one of the array operators 'append', 'prepend', 'delete', or 'insert'}", node, name)) return nil } @@ -231,9 +285,9 @@ func (m *Merger) mergeArray(orig []interface{}, n []interface{}, node string) [] } } else { // Index look-up based on key and name - key := modificationDefinitions[i].key - name := modificationDefinitions[i].name - delete := modificationDefinitions[i].delete + key := modificationDefinition.key + name := modificationDefinition.name + delete := modificationDefinition.listOp == listOpDelete // Sanity check original list, list must contain key/id based entries if err := canKeyMergeArray("original", result, node, key); err != nil { @@ -245,13 +299,13 @@ func (m *Merger) mergeArray(orig []interface{}, n []interface{}, node string) [] if delete == false { // Sanity check new list, list must contain key/id based entries - if err := canKeyMergeArray("new", modificationDefinitions[i].list, node, key); err != nil { + if err := canKeyMergeArray("new", modificationDefinition.list, node, key); err != nil { m.Errors.Append(err) return nil } // Since we have a way to identify indiviual entries based on their key/id, we can sanity check for possible duplicates - for _, entry := range modificationDefinitions[i].list { + for _, entry := range modificationDefinition.list { obj := entry.(map[interface{}]interface{}) entryName := obj[key].(string) if getIndexOfEntry(result, key, entryName) > 0 { @@ -261,7 +315,7 @@ func (m *Merger) mergeArray(orig []interface{}, n []interface{}, node string) [] } } else { // Sanity check for delete operation, ensure no orphan entries follow the operator definition - if len(modificationDefinitions[i].list) > 0 { + if len(modificationDefinition.list) > 0 { m.Errors.Append(ansi.Errorf("@m{%s}: @R{item in array directly after} @c{(( delete %s \"%s\" ))} @r{must be one of the array operators 'append', 'prepend', 'delete', or 'insert'}", node, key, name)) return nil } @@ -276,19 +330,19 @@ func (m *Merger) mergeArray(orig []interface{}, n []interface{}, node string) [] } // If after is specified, add one to the index to actually put the entry where it is expected - if modificationDefinitions[i].relative == "after" { + if modificationDefinition.relative == "after" { idx++ } // Back out if idx is smaller than 0, or greater than the length (for inserts), or greater/equal than the length (for deletes) - if (idx < 0) || (modificationDefinitions[i].delete == false && idx > len(result)) || (modificationDefinitions[i].delete == true && idx >= len(result)) { + if (idx < 0) || (modificationDefinition.listOp != listOpDelete && idx > len(result)) || (modificationDefinition.listOp == listOpDelete && idx >= len(result)) { m.Errors.Append(ansi.Errorf("@m{%s}: @R{unable to modify the list, because specified index} @c{%d} @R{is out of bounds}", node, idx)) return nil } - if modificationDefinitions[i].delete == false { - DEBUG("%s: inserting %d new elements to existing array at index %d", node, len(modificationDefinitions[i].list), idx) - result = insertIntoList(result, idx, modificationDefinitions[i].list) + if modificationDefinition.listOp != listOpDelete { + DEBUG("%s: inserting %d new elements to existing array at index %d", node, len(modificationDefinition.list), idx) + result = insertIntoList(result, idx, modificationDefinition.list) } else { DEBUG("%s: deleting element at array index %d", node, idx) result = deleteIndexFromList(result, idx) @@ -296,7 +350,6 @@ func (m *Merger) mergeArray(orig []interface{}, n []interface{}, node string) [] } return result - } // The magic which chooses to merge, append, or inline based on the contents of @@ -304,10 +357,8 @@ func (m *Merger) mergeArray(orig []interface{}, n []interface{}, node string) [] func (m *Merger) mergeArrayDefault(orig []interface{}, n []interface{}, node string) []interface{} { DEBUG("%s: performing index-based array merge", node) var err error - key := "name" - if os.Getenv("DEFAULT_ARRAY_MERGE_KEY") != "" { - key = os.Getenv("DEFAULT_ARRAY_MERGE_KEY") - } + key := getDefaultIdentifierKey() + if err = canKeyMergeArray("original", orig, node, key); err == nil { if err = canKeyMergeArray("new", n, node, key); err == nil { return m.mergeArrayByKey(orig, n, node, key) @@ -394,28 +445,22 @@ func (m *Merger) mergeArrayByKey(orig []interface{}, n []interface{}, node strin return merged } -func shouldInlineMergeArray(obj []interface{}) bool { - if len(obj) >= 1 && obj[0] != nil && reflect.TypeOf(obj[0]).Kind() == reflect.String { - re := regexp.MustCompile("^\\Q((\\E\\s*inline\\s*\\Q))\\E$") - if re.MatchString(obj[0].(string)) { - return true - } - } - return false -} - -//Returns a list of ModificationDefinition objects with information on which -// array operations to apply to which entries. The first object in the returned -// will always represent the default merge behavior. +// getArrayModifications returns a list of ModificationDefinition objects with +// information on which array operations to apply to which entries. The first +// object in the returned will always represent the default merge behavior. func getArrayModifications(obj []interface{}, simpleList bool) []ModificationDefinition { + // Starts with an entry representing the default merge behavior + result := []ModificationDefinition{ModificationDefinition{listOp: listOpMergeDefault}} - //Starts with an entry representing the default merge behavior - result := []ModificationDefinition{ModificationDefinition{defaultMerge: true}} - //easy shortcircuit + // easy shortcircuit if len(obj) == 0 { return result } + mergeRegEx := regexp.MustCompile("^\\Q((\\E\\s*merge\\s*\\Q))\\E$") + mergeOnKeyRegEx := regexp.MustCompile("^\\Q((\\E\\s*merge\\s+(on)\\s+(.+)\\s*\\Q))\\E$") + replaceRegEx := regexp.MustCompile("^\\Q((\\E\\s*replace\\s*\\Q))\\E$") + inlineRegEx := regexp.MustCompile("^\\Q((\\E\\s*inline\\s*\\Q))\\E$") appendRegEx := regexp.MustCompile("^\\Q((\\E\\s*append\\s*\\Q))\\E$") prependRegEx := regexp.MustCompile("^\\Q((\\E\\s*prepend\\s*\\Q))\\E$") insertByIdxRegEx := regexp.MustCompile("^\\Q((\\E\\s*insert\\s+(after|before)\\s+(\\d+)\\s*\\Q))\\E$") @@ -430,12 +475,35 @@ func getArrayModifications(obj []interface{}, simpleList bool) []ModificationDef case !isString: //Do absolutely nothing + case mergeRegEx.MatchString(e): // check for (( merge )) + result = append(result, ModificationDefinition{listOp: listOpMergeOnKey}) + continue + + case mergeOnKeyRegEx.MatchString(e): // check for (( merge on "key" )) + /* #0 is the whole string, + * #1 is string 'on' + * #2 is the named-entry identifying key + */ + if captures := mergeOnKeyRegEx.FindStringSubmatch(e); len(captures) == 3 { + key := strings.TrimSpace(captures[2]) + result = append(result, ModificationDefinition{listOp: listOpMergeOnKey, key: key}) + continue + } + + case inlineRegEx.MatchString(e): // check for (( inline )) + result = append(result, ModificationDefinition{listOp: listOpMergeInline}) + continue + + case replaceRegEx.MatchString(e): // check for (( replace )) + result = append(result, ModificationDefinition{listOp: listOpReplace}) + continue + case appendRegEx.MatchString(e): // check for (( append )) - result = append(result, ModificationDefinition{index: -1}) + result = append(result, ModificationDefinition{listOp: listOpInsert, index: -1}) continue case prependRegEx.MatchString(e): // check for (( prepend )) - result = append(result, ModificationDefinition{index: 0}) + result = append(result, ModificationDefinition{listOp: listOpInsert, index: 0}) continue case insertByIdxRegEx.MatchString(e): // check for (( insert ... )) @@ -447,7 +515,7 @@ func getArrayModifications(obj []interface{}, simpleList bool) []ModificationDef relative := strings.TrimSpace(captures[1]) position := strings.TrimSpace(captures[2]) if idx, err := strconv.Atoi(position); err == nil { - result = append(result, ModificationDefinition{index: idx, relative: relative}) + result = append(result, ModificationDefinition{listOp: listOpInsert, index: idx, relative: relative}) continue } } @@ -464,12 +532,13 @@ func getArrayModifications(obj []interface{}, simpleList bool) []ModificationDef name := strings.TrimSpace(captures[3]) if key == "" { - key = "name" + key = getDefaultIdentifierKey() } - result = append(result, ModificationDefinition{relative: relative, key: key, name: name}) + result = append(result, ModificationDefinition{listOp: listOpInsert, relative: relative, key: key, name: name}) continue } + case deleteByIdxRegEx.MatchString(e): // check for (( delete )) /* #0 is the whole string, * #1 is idx @@ -477,10 +546,11 @@ func getArrayModifications(obj []interface{}, simpleList bool) []ModificationDef if captures := deleteByIdxRegEx.FindStringSubmatch(e); len(captures) == 2 { position := strings.TrimSpace(captures[1]) if idx, err := strconv.Atoi(position); err == nil { - result = append(result, ModificationDefinition{index: idx, delete: true}) + result = append(result, ModificationDefinition{listOp: listOpDelete, index: idx}) continue } } + case deleteByNameRegEx.MatchString(e): // check for (( delete "" )) /* #0 is the whole string, * #1 contains the optional '' string @@ -496,12 +566,13 @@ func getArrayModifications(obj []interface{}, simpleList bool) []ModificationDef } if !simpleList && key == "" { - key = "name" + key = getDefaultIdentifierKey() } - result = append(result, ModificationDefinition{key: key, name: name, delete: true}) + result = append(result, ModificationDefinition{listOp: listOpDelete, key: key, name: name}) continue } + case deleteByNameUnquotedRegEx.MatchString(e): // check for (( delete "" )) /* #0 is the whole string, * #1 contains the optional '' string @@ -519,18 +590,19 @@ func getArrayModifications(obj []interface{}, simpleList bool) []ModificationDef if name == "" { name = key if !simpleList { - key = "name" + key = getDefaultIdentifierKey() } else { key = "" } } - result = append(result, ModificationDefinition{key: key, name: name, delete: true}) + result = append(result, ModificationDefinition{listOp: listOpDelete, key: key, name: name}) continue } } lastResultIdx := len(result) - 1 + // Add the current entry to the 'current' modification definition record (gathering the list) result[lastResultIdx].list = append(result[lastResultIdx].list, entry) } @@ -559,10 +631,7 @@ func isSimpleList(list []interface{}) bool { } func shouldKeyMergeArray(obj []interface{}) (bool, string) { - key := "name" - if os.Getenv("DEFAULT_ARRAY_MERGE_KEY") != "" { - key = os.Getenv("DEFAULT_ARRAY_MERGE_KEY") - } + key := getDefaultIdentifierKey() if len(obj) >= 1 && obj[0] != nil && reflect.TypeOf(obj[0]).Kind() == reflect.String { re := regexp.MustCompile("^\\Q((\\E\\s*merge(?:\\s+on\\s+(.*?))?\\s*\\Q))\\E$") @@ -606,17 +675,6 @@ func canKeyMergeArray(disp string, array []interface{}, node string, key string) return nil } -func shouldReplaceArray(obj []interface{}) bool { - if len(obj) >= 1 && obj[0] != nil && reflect.TypeOf(obj[0]).Kind() == reflect.String { - re := regexp.MustCompile(`^\Q((\E\s*replace\s*\Q))\E$`) - - if re.MatchString(obj[0].(string)) { - return true - } - } - return false -} - func getIndexOfSimpleEntry(list []interface{}, name string) int { for i, entry := range list { switch entry.(type) { diff --git a/vendor/github.com/geofffranks/spruce/merge_test.go b/vendor/github.com/geofffranks/spruce/merge_test.go index 531e166..adbe18e 100644 --- a/vendor/github.com/geofffranks/spruce/merge_test.go +++ b/vendor/github.com/geofffranks/spruce/merge_test.go @@ -11,68 +11,6 @@ import ( "github.com/starkandwayne/goutils/tree" ) -func TestShouldReplaceArray(t *testing.T) { - Convey("We should replace arrays", t, func() { - Convey("If the element is a string with the right append token", func() { - So(shouldReplaceArray([]interface{}{"(( replace ))", "stuff"}), ShouldBeTrue) - }) - Convey("But not if the element is a string with the wrong token", func() { - So(shouldReplaceArray([]interface{}{"not a magic token"}), ShouldBeFalse) - }) - Convey("But not if the element is not a string", func() { - So(shouldReplaceArray([]interface{}{42}), ShouldBeFalse) - }) - Convey("But not if the slice has no elements", func() { - So(shouldReplaceArray([]interface{}{}), ShouldBeFalse) - }) - Convey("Is whitespace agnostic", func() { - Convey("No surrounding whitespace", func() { - yes := shouldReplaceArray([]interface{}{"((replace))"}) - So(yes, ShouldBeTrue) - }) - Convey("Surrounding tabs", func() { - yes := shouldReplaceArray([]interface{}{"(( replace ))"}) - So(yes, ShouldBeTrue) - }) - Convey("Multiple surrounding whitespaces", func() { - yes := shouldReplaceArray([]interface{}{"(( replace ))"}) - So(yes, ShouldBeTrue) - }) - }) - }) -} - -func TestShouldInlineMergeArray(t *testing.T) { - Convey("We should inline merge arrays", t, func() { - Convey("If the element is a string with the right inline-merge token", func() { - So(shouldInlineMergeArray([]interface{}{"(( inline ))", "stuff"}), ShouldBeTrue) - }) - Convey("But not if the element is a string with the wrong token", func() { - So(shouldInlineMergeArray([]interface{}{"not a magic token"}), ShouldBeFalse) - }) - Convey("But not if the element is not a string", func() { - So(shouldInlineMergeArray([]interface{}{42}), ShouldBeFalse) - }) - Convey("But not if the slice has no elements", func() { - So(shouldInlineMergeArray([]interface{}{}), ShouldBeFalse) - }) - Convey("Is whitespace agnostic", func() { - Convey("No surrounding whitespace", func() { - yes := shouldInlineMergeArray([]interface{}{"((inline))"}) - So(yes, ShouldBeTrue) - }) - Convey("Surrounding tabs", func() { - yes := shouldInlineMergeArray([]interface{}{"(( inline ))"}) - So(yes, ShouldBeTrue) - }) - Convey("Multiple surrounding whitespaces", func() { - yes := shouldInlineMergeArray([]interface{}{"(( inline ))"}) - So(yes, ShouldBeTrue) - }) - }) - }) -} - func TestShouldKeyMergeArrayOfHashes(t *testing.T) { Convey("We should key-based merge arrays of hashes", t, func() { Convey("If the element is a string with the right key-merge token", func() { @@ -168,21 +106,95 @@ func TestGetArrayModifications(t *testing.T) { } shouldBeDelete := func(actual interface{}, _ ...interface{}) string { - if !actual.(ModificationDefinition).delete { + if actual.(ModificationDefinition).listOp != listOpDelete { return "Result doesn't have marker for delete operation" } return "" } shouldBeDefault := func(actual interface{}, _ ...interface{}) string { - if actual.(ModificationDefinition).defaultMerge { + if actual.(ModificationDefinition).listOp == listOpMergeDefault { return "" } return "Expected defaultMerge to be true" } + shouldBeMergeOnKey := func(actual interface{}, _ ...interface{}) string { + switch actual.(ModificationDefinition).listOp { + case listOpMergeOnKey: + return "" + + default: + return "Expected list operation to be 'merge on key'" + } + } + + shouldBeReplace := func(actual interface{}, _ ...interface{}) string { + switch actual.(ModificationDefinition).listOp { + case listOpReplace: + return "" + + default: + return "Expected list operation to be 'replace'" + } + } + Convey("Should recognize string patterns for", t, func() { + Convey("(( merge ))", func() { + //merge test cases go here + for input, shouldMatch := range map[string]bool{ + "(( merge ))": true, + "((merge))": true, + "(( merge ))": true, + "(( merge ))": true, + "(( merge))": true, + "(( notmerge ))": false, + "(( mergenot ))": false, + "(( not even merge ))": false, + "(( somethingelse ))": false, + } { + Convey(fmt.Sprintf("with case %s", input), func() { + results := getArrayModifications([]interface{}{input}, false) + if shouldMatch { + So(results, ShouldHaveLength, 2) + So(results[0], shouldBeDefault) + So(results[1], shouldBeMergeOnKey) + } else { + So(results, ShouldHaveLength, 1) + So(results[0], shouldBeDefault) + } + }) + } + }) + + Convey("(( replace ))", func() { + //replace test cases go here + for input, shouldMatch := range map[string]bool{ + "(( replace ))": true, + "((replace))": true, + "(( replace ))": true, + "(( replace ))": true, + "(( replace))": true, + "(( notreplace ))": false, + "(( replacenot ))": false, + "(( not even replace ))": false, + "(( somethingelse ))": false, + } { + Convey(fmt.Sprintf("with case %s", input), func() { + results := getArrayModifications([]interface{}{input}, false) + if shouldMatch { + So(results, ShouldHaveLength, 2) + So(results[0], shouldBeDefault) + So(results[1], shouldBeReplace) + } else { + So(results, ShouldHaveLength, 1) + So(results[0], shouldBeDefault) + } + }) + } + }) + Convey("(( append ))", func() { //append test cases go here for input, shouldMatch := range map[string]bool{ @@ -410,7 +422,7 @@ func TestGetArrayModifications(t *testing.T) { Convey("Don't return an insert if index is obviously out of bounds", t, func() { results := getArrayModifications([]interface{}{"(( insert before -1 ))", "stuff"}, false) So(results, ShouldHaveLength, 1) //Just the default merge - So(results[0].defaultMerge, ShouldBeTrue) + So(results[0].listOp, ShouldEqual, listOpMergeDefault) }) Convey("If there are multiple insert token with after/before, different key names, and names (only technical usecase)", t, func() { @@ -444,7 +456,7 @@ func TestGetArrayModifications(t *testing.T) { Convey("Can specify operators without one at the 0th index", t, func() { results := getArrayModifications([]interface{}{"foo", "(( append ))", "stuff"}, false) So(results, ShouldHaveLength, 2) - So(results[0].defaultMerge, ShouldBeTrue) + So(results[0].listOp, ShouldEqual, listOpMergeDefault) So(results[1], shouldBeAppend) So(results[0].list, ShouldHaveLength, 1) So(results[1].list, ShouldHaveLength, 1) diff --git a/vendor/github.com/geofffranks/spruce/op_calc.go b/vendor/github.com/geofffranks/spruce/op_calc.go index 6e2c1e3..e65844f 100644 --- a/vendor/github.com/geofffranks/spruce/op_calc.go +++ b/vendor/github.com/geofffranks/spruce/op_calc.go @@ -100,11 +100,11 @@ func (CalcOperator) Run(ev *Evaluator, args []*Expr) (*Response, error) { func searchForCursors(input string) ([]*tree.Cursor, error) { result := []*tree.Cursor{} - // Search for sub-strings that contain the path seperator dot character + // Search for sub-strings that contain the path separator dot character // https://regex101.com/r/TIEyak/1 (to delete the URL use https://regex101.com/delete/fPbxosYXWzBPYaNdL5YcPpj3) regexp := regexp.MustCompile(`(\w+|-)\.(\w+|-|\.)+`) candidates := regexp.FindAllString(input, -1) - DEBUG(" strings found containing the path seperator: %v", strings.Join(candidates, ", ")) + DEBUG(" strings found containing the path separator: %v", strings.Join(candidates, ", ")) // If it is a path, it can be parsed (parse errors will be ignored) for _, candidate := range candidates { diff --git a/vendor/github.com/geofffranks/spruce/op_join.go b/vendor/github.com/geofffranks/spruce/op_join.go index a0de38d..f6c034f 100644 --- a/vendor/github.com/geofffranks/spruce/op_join.go +++ b/vendor/github.com/geofffranks/spruce/op_join.go @@ -107,11 +107,11 @@ func (JoinOperator) Run(ev *Evaluator, args []*Expr) (*Response, error) { return nil, ansi.Errorf("too few arguments supplied to @c{(( join ... ))}") } - var seperator string + var separator string var list []string for i, arg := range args { - if i == 0 { // argument #0: seperator + if i == 0 { // argument #0: separator sep, err := arg.Resolve(ev.Tree) if err != nil { DEBUG(" [%d]: resolution failed\n error: %s", i, err) @@ -119,12 +119,12 @@ func (JoinOperator) Run(ev *Evaluator, args []*Expr) (*Response, error) { } if sep.Type != Literal { - DEBUG(" [%d]: unsupported type for join operator seperator argument: '%v'", i, sep) - return nil, fmt.Errorf("join operator only accepts literal argument for the seperator") + DEBUG(" [%d]: unsupported type for join operator separator argument: '%v'", i, sep) + return nil, fmt.Errorf("join operator only accepts literal argument for the separator") } - DEBUG(" [%d]: list seperator will be: %s", i, sep) - seperator = sep.Literal.(string) + DEBUG(" [%d]: list separator will be: %s", i, sep) + separator = sep.Literal.(string) } else { // argument #1..n: list, or literal ref, err := arg.Resolve(ev.Tree) @@ -181,10 +181,10 @@ func (JoinOperator) Run(ev *Evaluator, args []*Expr) (*Response, error) { } // finally, join and return - DEBUG(" joined list: %s", strings.Join(list, seperator)) + DEBUG(" joined list: %s", strings.Join(list, separator)) return &Response{ Type: Replace, - Value: strings.Join(list, seperator), + Value: strings.Join(list, separator), }, nil } diff --git a/vendor/github.com/geofffranks/spruce/operator_test.go b/vendor/github.com/geofffranks/spruce/operator_test.go index 95be0b6..f5df64e 100644 --- a/vendor/github.com/geofffranks/spruce/operator_test.go +++ b/vendor/github.com/geofffranks/spruce/operator_test.go @@ -1496,13 +1496,13 @@ meta: So(r, ShouldBeNil) }) - Convey("throws an error when seperator argument is not a literal", func() { + Convey("throws an error when separator argument is not a literal", func() { r, err := op.Run(ev, []*Expr{ ref("meta.emptylist"), ref("meta.authorities"), }) So(err, ShouldNotBeNil) - So(err.Error(), ShouldContainSubstring, "join operator only accepts literal argument for the seperator") + So(err.Error(), ShouldContainSubstring, "join operator only accepts literal argument for the separator") So(r, ShouldBeNil) })