From b20587fba0d175765297a5366aaf7c6740f75058 Mon Sep 17 00:00:00 2001 From: Eugene Dementiev Date: Mon, 10 Dec 2018 13:56:21 +1300 Subject: [PATCH] Implement plain parameters. Fixes #4 --- Gopkg.lock | 40 +++++++++++++++++++-- README.md | 26 +++++++++----- cmd/dotenv.go | 2 +- cmd/print.go | 2 +- cmd/root.go | 23 ++++++++----- cmd/run.go | 2 +- ssm/parameters.go | 88 ++++++++++++++++++++++++++++++++++++++++++++++- 7 files changed, 161 insertions(+), 22 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index c54445e..58dfb91 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -2,12 +2,15 @@ [[projects]] + digest = "1:40261f597e43f31aaf9f8658b85c082773217943056f1be137635e769c0798c9" name = "github.com/apex/log" packages = ["."] + pruneopts = "UT" revision = "0296d6eb16bb28f8a0c55668affcf4876dc269be" version = "v1.0.0" [[projects]] + digest = "1:d77b89a7b72b33e5a3b0ac2334484a06bafba55df4bfc24cea325962f3254819" name = "github.com/aws/aws-sdk-go" packages = [ "aws", @@ -37,79 +40,112 @@ "private/protocol/rest", "private/protocol/xml/xmlutil", "service/ssm", - "service/sts" + "service/sts", ] + pruneopts = "UT" revision = "9b0098a71f6d4d473a26ec8ad3c2feaac6eb1da6" version = "v1.13.32" [[projects]] branch = "master" + digest = "1:f7b937d8308793c2ceb87ab884f3ae5092d7ed78889147667de86b871de2ebcc" name = "github.com/buildkite/interpolate" packages = ["."] + pruneopts = "UT" revision = "c1c376f870d259871fcc9bf3bcfd4889b999ed1a" [[projects]] + digest = "1:ad12727c184fc7e33768f5fe239d8f767a3b60264541067ac2f23b9aca013e06" name = "github.com/go-ini/ini" packages = ["."] + pruneopts = "UT" revision = "ace140f73450505f33e8b8418216792275ae82a7" version = "v1.35.0" [[projects]] branch = "master" + digest = "1:07671f8997086ed115824d1974507d2b147d1e0463675ea5dbf3be89b1c2c563" name = "github.com/hashicorp/errwrap" packages = ["."] + pruneopts = "UT" revision = "7554cd9344cec97297fa6649b055a8c98c2a1e55" [[projects]] branch = "master" + digest = "1:e5048c5da80697be2fcdecc944e29d2999e01fd7f48b643168443209779f3463" name = "github.com/hashicorp/go-multierror" packages = ["."] + pruneopts = "UT" revision = "b7773ae218740a7be65057fc60b366a49b538a44" [[projects]] + digest = "1:b39dba8363bf11da3cddcc31c5a9c39f17fba8a5317ae0836ea4530bae3b4866" name = "github.com/imdario/mergo" packages = ["."] + pruneopts = "UT" revision = "9d5f1277e9a8ed20c3684bda8fde67c05628518c" version = "v0.3.4" [[projects]] + digest = "1:870d441fe217b8e689d7949fef6e43efbc787e50f200cb1e70dbca9204a1d6be" name = "github.com/inconshreveable/mousetrap" packages = ["."] + pruneopts = "UT" revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75" version = "v1.0" [[projects]] + digest = "1:e22af8c7518e1eab6f2eab2b7d7558927f816262586cd6ed9f349c97a6c285c4" name = "github.com/jmespath/go-jmespath" packages = ["."] + pruneopts = "UT" revision = "0b12d6b5" [[projects]] + digest = "1:ecd9aa82687cf31d1585d4ac61d0ba180e42e8a6182b85bd785fcca8dfeefc1b" name = "github.com/joho/godotenv" packages = ["."] + pruneopts = "UT" revision = "23d116af351c84513e1946b527c88823e476be13" version = "v1.3.0" [[projects]] + digest = "1:40e195917a951a8bf867cd05de2a46aaf1806c50cf92eebf4c16f78cd196f747" name = "github.com/pkg/errors" packages = ["."] + pruneopts = "UT" revision = "645ef00459ed84a119197bfb8d8205042c6df63d" version = "v0.8.0" [[projects]] + digest = "1:982714fa3eaafd4251f5ccf3d133b706ad742b86c195d594e1a86050a02f87a8" name = "github.com/spf13/cobra" packages = ["."] + pruneopts = "UT" revision = "a1f051bc3eba734da4772d60e2d677f47cf93ef4" version = "v0.0.2" [[projects]] + digest = "1:9424f440bba8f7508b69414634aef3b2b3a877e522d8a4624692412805407bb7" name = "github.com/spf13/pflag" packages = ["."] + pruneopts = "UT" revision = "583c0c0531f06d5278b7d917446061adc344b5cd" version = "v1.0.1" [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "7cefb712d6c13a4e630e4c69a83a4caaa0526b6dcd02db7f0007edc71e82cba2" + input-imports = [ + "github.com/apex/log", + "github.com/aws/aws-sdk-go/aws", + "github.com/aws/aws-sdk-go/aws/session", + "github.com/aws/aws-sdk-go/service/ssm", + "github.com/buildkite/interpolate", + "github.com/hashicorp/go-multierror", + "github.com/imdario/mergo", + "github.com/joho/godotenv", + "github.com/spf13/cobra", + ] solver-name = "gps-cdcl" solver-version = 1 diff --git a/README.md b/README.md index c0be060..d758679 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ All parameters must be in JSON format, i.e.: } ``` -If a few parameters are specified, all JSON entities will be read and merged into one, overriding existing keys, i.e. +If several parameters are specified, all JSON entities will be read and merged into one, overriding existing keys, i.e. Parameter one: ``` @@ -39,6 +39,10 @@ The result will be merged as this: } ``` +One can also specify `--plain-name` and `--plain-path` command line options to read _plain_ parameters that are not in JSON format. +`ssm-parent` takes the value as is, and constructs a key name from the `basename parameter`, + i.e. a SSM Parameter `/project/environment/myParameter` with value `supervalue` will be exported as `myParameter=supervalue`. + ### How to use @@ -48,7 +52,10 @@ That should be pretty self-explanatory. SSM-Parent is a docker entrypoint. It gets specified parameters (possibly secret) from AWS SSM Parameter Store, -then exports them to the underlying process. +then exports them to the underlying process. Or creates a .env file to be consumed by an application. + +It reads parameters in the following order: path->name->plain-path->plain-name. +So that every rightmost parameter overrides the previous one. Usage: ssm-parent [command] @@ -60,12 +67,15 @@ Available Commands: run Runs the specified command Flags: - -e, --expand Expand arguments and values using shell-style syntax - -h, --help help for ssm-parent - -n, --name stringArray Name of the SSM parameter to retrieve. Can be specified multiple times. - -p, --path stringArray Path to a SSM parameter. Can be specified multiple times. - -r, --recursive Walk through the provided SSM paths recursively. - -s, --strict Strict mode. Fail if found less parameters than number of names. + -e, --expand Expand arguments and values using shell-style syntax + -h, --help help for ssm-parent + -n, --name stringArray Name of the SSM parameter to retrieve. Expects JSON in the value. Can be specified multiple times. + -p, --path stringArray Path to a SSM parameter. Expects JSON in the value. Can be specified multiple times. + --plain-name stringArray Name of the SSM parameter to retrieve. Expects actual parameter in the value. Can be specified multiple times. + --plain-path stringArray Path to a SSM parameter. Expects actual parameter in the value. Can be specified multiple times. + -r, --recursive Walk through the provided SSM paths recursively. + -s, --strict Strict mode. Fail if found less parameters than number of names. + --version version for ssm-parent Use "ssm-parent [command] --help" for more information about a command. ``` diff --git a/cmd/dotenv.go b/cmd/dotenv.go index fef8ec0..9b5736a 100644 --- a/cmd/dotenv.go +++ b/cmd/dotenv.go @@ -16,7 +16,7 @@ var dotenvCmd = &cobra.Command{ Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { megamap := make(map[string]string) - parameters, err := ssm.GetParameters(names, paths, expand, strict, recursive) + parameters, err := ssm.GetParameters(names, paths, plainNames, plainPaths, expand, strict, recursive) if err != nil { log.WithError(err).Fatal("Can't get parameters") } diff --git a/cmd/print.go b/cmd/print.go index d3b4030..0803252 100644 --- a/cmd/print.go +++ b/cmd/print.go @@ -17,7 +17,7 @@ var printCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { megamap := make(map[string]string) - parameters, err := ssm.GetParameters(names, paths, expand, strict, recursive) + parameters, err := ssm.GetParameters(names, paths, plainNames, plainPaths, expand, strict, recursive) if err != nil { log.WithError(err).Fatal("Can't get parameters") } diff --git a/cmd/root.go b/cmd/root.go index c19e7bb..d049cff 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -8,11 +8,13 @@ import ( ) var ( - paths []string - names []string - recursive bool - strict bool - expand bool + paths []string + names []string + plainPaths []string + plainNames []string + recursive bool + strict bool + expand bool ) // rootCmd represents the base command when called without any subcommands @@ -22,7 +24,10 @@ var rootCmd = &cobra.Command{ Long: `SSM-Parent is a docker entrypoint. It gets specified parameters (possibly secret) from AWS SSM Parameter Store, -then exports them to the underlying process. +then exports them to the underlying process. Or creates a .env file to be consumed by an application. + +It reads parameters in the following order: path->name->plain-path->plain-name. +So that every rightmost parameter overrides the previous one. `, } @@ -35,8 +40,10 @@ func Execute(version string) { } func init() { - rootCmd.PersistentFlags().StringArrayVarP(&paths, "path", "p", []string{}, "Path to a SSM parameter. Can be specified multiple times.") - rootCmd.PersistentFlags().StringArrayVarP(&names, "name", "n", []string{}, "Name of the SSM parameter to retrieve. Can be specified multiple times.") + rootCmd.PersistentFlags().StringArrayVarP(&paths, "path", "p", []string{}, "Path to a SSM parameter. Expects JSON in the value. Can be specified multiple times.") + rootCmd.PersistentFlags().StringArrayVarP(&names, "name", "n", []string{}, "Name of the SSM parameter to retrieve. Expects JSON in the value. Can be specified multiple times.") + rootCmd.PersistentFlags().StringArrayVarP(&plainPaths, "plain-path", "", []string{}, "Path to a SSM parameter. Expects actual parameter in the value. Can be specified multiple times.") + rootCmd.PersistentFlags().StringArrayVarP(&plainNames, "plain-name", "", []string{}, "Name of the SSM parameter to retrieve. Expects actual parameter in the value. Can be specified multiple times.") rootCmd.PersistentFlags().BoolVarP(&recursive, "recursive", "r", false, "Walk through the provided SSM paths recursively.") rootCmd.PersistentFlags().BoolVarP(&strict, "strict", "s", false, "Strict mode. Fail if found less parameters than number of names.") rootCmd.PersistentFlags().BoolVarP(&expand, "expand", "e", false, "Expand arguments and values using shell-style syntax") diff --git a/cmd/run.go b/cmd/run.go index 0ef22ec..8a48091 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -21,7 +21,7 @@ var runCmd = &cobra.Command{ var cmdArgs []string megamap := make(map[string]string) - parameters, err := ssm.GetParameters(names, paths, expand, strict, recursive) + parameters, err := ssm.GetParameters(names, paths, plainNames, plainPaths, expand, strict, recursive) if err != nil { log.WithError(err).Fatal("Can't get parameters") } diff --git a/ssm/parameters.go b/ssm/parameters.go index 1187561..c6c2de4 100644 --- a/ssm/parameters.go +++ b/ssm/parameters.go @@ -3,6 +3,7 @@ package ssm import ( "encoding/json" "fmt" + goPath "path" "github.com/apex/log" "github.com/aws/aws-sdk-go/aws" @@ -57,6 +58,30 @@ func getJsonSSMParametersByPaths(paths []string, strict, recursive bool) (parame return } +func getPlainSSMParametersByPaths(paths []string, strict, recursive bool) (parameters []map[string]string, err error) { + err = makeSession() + if err != nil { + log.WithError(err).Fatal("Can't create session") // fail early here + } + s := ssm.New(localSession) + for _, path := range paths { + response, innerErr := s.GetParametersByPath(&ssm.GetParametersByPathInput{ + Path: aws.String(path), + WithDecryption: aws.Bool(true), + Recursive: aws.Bool(recursive), + }) + if innerErr != nil { + err = multierror.Append(err, fmt.Errorf("Can't get parameters from path '%s': %s", path, innerErr)) + } + for _, parameter := range response.Parameters { + values := make(map[string]string) + values[goPath.Base(aws.StringValue(parameter.Name))] = aws.StringValue(parameter.Value) + parameters = append(parameters, values) + } + } + return +} + func getJsonSSMParameters(names []string, strict bool) (parameters []map[string]string, err error) { err = makeSession() if err != nil { @@ -93,13 +118,51 @@ func getJsonSSMParameters(names []string, strict bool) (parameters []map[string] return } -func GetParameters(names, paths []string, expand, strict, recursive bool) (parameters []map[string]string, err error) { +func getPlainSSMParameters(names []string, strict bool) (parameters []map[string]string, err error) { + err = makeSession() + if err != nil { + log.WithError(err).Fatal("Can't create session") // fail early here + } + s := ssm.New(localSession) + response, err := s.GetParameters(&ssm.GetParametersInput{ + Names: aws.StringSlice(names), + WithDecryption: aws.Bool(true), + }) + if err != nil { + return nil, err + } + if len(response.Parameters) < len(names) { + if strict { + err = multierror.Append(err, fmt.Errorf("Found %d parameters from %d names", len(response.Parameters), len(names))) + } else { + var found []string + for _, f := range response.Parameters { + found = append(found, *f.Name) + } + diff := stringSliceDifference(names, found) + log.WithFields(log.Fields{"missing_names": diff}).Warn("Some parameters have not been found") + } + } + for _, parameter := range response.Parameters { + values := make(map[string]string) + values[goPath.Base(aws.StringValue(parameter.Name))] = aws.StringValue(parameter.Value) + parameters = append(parameters, values) + } + return +} + +func GetParameters(names, paths, plainNames, plainPaths []string, expand, strict, recursive bool) (parameters []map[string]string, err error) { localNames := names localPaths := paths + localPlainNames := plainNames + localPlainPaths := plainPaths if expand { localNames = ExpandArgs(names) localPaths = ExpandArgs(paths) + localPlainNames = ExpandArgs(plainNames) + localPlainPaths = ExpandArgs(plainPaths) } + if len(localPaths) > 0 { parametersFromPaths, err := getJsonSSMParametersByPaths(localPaths, strict, recursive) if err != nil { @@ -122,5 +185,28 @@ func GetParameters(names, paths []string, expand, strict, recursive bool) (param parameters = append(parameters, parameter) } } + if len(localPlainPaths) > 0 { + parametersFromPlainPaths, err := getPlainSSMParametersByPaths(localPlainPaths, strict, recursive) + if err != nil { + log.WithError(err).WithFields( + log.Fields{"plain_paths": localPaths}, + ).Fatal("Can't get plain parameters by paths") + } + for _, parameter := range parametersFromPlainPaths { + parameters = append(parameters, parameter) + } + } + + if len(localPlainNames) > 0 { + parametersFromPlainNames, err := getPlainSSMParameters(localPlainNames, strict) + if err != nil { + log.WithError(err).WithFields( + log.Fields{"plain_names": localNames}, + ).Fatal("Can't get plain parameters by names") + } + for _, parameter := range parametersFromPlainNames { + parameters = append(parameters, parameter) + } + } return }