From af252c1048069d39a556c77624fc0af6608f59de Mon Sep 17 00:00:00 2001 From: Eugene Dementyev Date: Mon, 31 Jan 2022 12:03:49 +1300 Subject: [PATCH] Add selective parameter expansion flags and implementation --- .github/workflows/build_test.yml | 37 ++++++++++++++++++ cmd/dotenv.go | 3 ++ cmd/print.go | 3 ++ cmd/root.go | 8 +++- cmd/run.go | 3 ++ ssm/expand.go | 52 ++++++++++++++++++++++++++ ssm/expand_test.go | 64 ++++++++++++++++++++++++++++++++ ssm/parameters.go | 16 ++++---- ssm/util.go | 23 ------------ 9 files changed, 178 insertions(+), 31 deletions(-) create mode 100644 .github/workflows/build_test.yml create mode 100644 ssm/expand.go create mode 100644 ssm/expand_test.go diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml new file mode 100644 index 0000000..cc629cc --- /dev/null +++ b/.github/workflows/build_test.yml @@ -0,0 +1,37 @@ +name: "Test the build" + +on: + push: + pull_request: + +jobs: + test: + name: test if ssm-parent can be built + runs-on: ubuntu-latest + steps: + - + name: checkout + uses: actions/checkout@v2 + - + name: set up Go + uses: actions/setup-go@v1 + with: + go-version: 1.17.x + - + name: cache modules + uses: actions/cache@v2 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + - + name: download dependencies + run: go mod download + - + name: build the app + run: go build + - + name: test the app + run: go test -v ./... + diff --git a/cmd/dotenv.go b/cmd/dotenv.go index daa3560..aa8ca9d 100644 --- a/cmd/dotenv.go +++ b/cmd/dotenv.go @@ -27,6 +27,9 @@ var dotenvCmd = &cobra.Command{ viper.GetBool("expand"), viper.GetBool("strict"), viper.GetBool("recursive"), + viper.GetBool("expand-names"), + viper.GetBool("expand-paths"), + viper.GetStringSlice("expand-values"), ) if err != nil { log.WithError(err).Fatal("Can't get parameters") diff --git a/cmd/print.go b/cmd/print.go index 9bf4b35..96efc97 100644 --- a/cmd/print.go +++ b/cmd/print.go @@ -25,6 +25,9 @@ var printCmd = &cobra.Command{ viper.GetBool("expand"), viper.GetBool("strict"), viper.GetBool("recursive"), + viper.GetBool("expand-names"), + viper.GetBool("expand-paths"), + viper.GetStringSlice("expand-values"), ) if err != nil { log.WithError(err).Fatal("Can't marshal json") diff --git a/cmd/root.go b/cmd/root.go index 6322f07..fafa3a0 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -103,7 +103,10 @@ func init() { cobra.OnInitialize(initSettings) rootCmd.PersistentFlags().StringVarP(&config, "config", "c", "", "Path to the config file (optional). Allows to set transformations") rootCmd.PersistentFlags().BoolP("debug", "d", false, "Turn on debug logging") - rootCmd.PersistentFlags().BoolP("expand", "e", false, "Expand arguments and values using shell-style syntax") + rootCmd.PersistentFlags().BoolP("expand", "e", false, "Expand all arguments and values using shell-style syntax") + rootCmd.PersistentFlags().BoolP("expand-names", "", false, "Expand SSM names using shell-style syntax. The '--expand' does the same, but this flag is more selective") + rootCmd.PersistentFlags().BoolP("expand-paths", "", false, "Expand SSM paths using shell-style syntax. The '--expand' does the same, but this flag is more selective") + rootCmd.PersistentFlags().StringSliceP("expand-values", "", []string{}, "Expand SSM values using shell-style syntax. The '--expand' does the same, but this flag is more selective. Can be specified multiple times.") rootCmd.PersistentFlags().StringSliceP("path", "p", []string{}, "Path to a SSM parameter. Expects JSON in the value. Can be specified multiple times.") rootCmd.PersistentFlags().StringSliceP("name", "n", []string{}, "Name of the SSM parameter to retrieve. Expects JSON in the value. Can be specified multiple times.") rootCmd.PersistentFlags().StringSliceP("plain-path", "", []string{}, "Path to a SSM parameter. Expects actual parameter in the value. Can be specified multiple times.") @@ -113,6 +116,9 @@ func init() { viper.BindPFlag("debug", rootCmd.PersistentFlags().Lookup("debug")) viper.BindPFlag("expand", rootCmd.PersistentFlags().Lookup("expand")) + viper.BindPFlag("expand-names", rootCmd.PersistentFlags().Lookup("expand-names")) + viper.BindPFlag("expand-paths", rootCmd.PersistentFlags().Lookup("expand-paths")) + viper.BindPFlag("expand-values", rootCmd.PersistentFlags().Lookup("expand-values")) viper.BindPFlag("paths", rootCmd.PersistentFlags().Lookup("path")) viper.BindPFlag("names", rootCmd.PersistentFlags().Lookup("name")) viper.BindPFlag("plain-paths", rootCmd.PersistentFlags().Lookup("plain-path")) diff --git a/cmd/run.go b/cmd/run.go index ddd926e..30278fc 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -30,6 +30,9 @@ var runCmd = &cobra.Command{ viper.GetBool("expand"), viper.GetBool("strict"), viper.GetBool("recursive"), + viper.GetBool("expand-names"), + viper.GetBool("expand-paths"), + viper.GetStringSlice("expand-values"), ) if err != nil { log.WithError(err).Fatal("Can't get parameters") diff --git a/ssm/expand.go b/ssm/expand.go new file mode 100644 index 0000000..08c6eee --- /dev/null +++ b/ssm/expand.go @@ -0,0 +1,52 @@ +package ssm + +import ( + "fmt" + "strings" + + "github.com/buildkite/interpolate" +) + +// expandArgs expands arguments using env vars +func ExpandArgs(args []string) []string { + var expanded []string + for _, arg := range args { + arg = expandValue(arg) + expanded = append(expanded, arg) + } + return expanded +} + +// expandValue interpolates values using env vars +func expandValue(val string) string { + e, err := interpolate.Interpolate(env, val) + if err == nil { + return strings.TrimSpace(string(e)) + } + return val +} + +// expandParameters expands values using shell-like syntax +func expandParameters(parameters map[string]string, expand bool, expandValues []string) error { + + // if global expand is true then just it for all + if expand { + for key, value := range parameters { + parameters[key] = expandValue(value) + } + // can return early as we've done the job + return nil + } + // check if all values that we ask to expand present in the parameters + // otherwise, it's a configuration error + for _, val := range expandValues { + if _, ok := parameters[val]; !ok { + return fmt.Errorf("env var %s is present in the expand-values but doesn't exist in the environment", val) + } else { + // if the var is present we expand it + parameters[val] = expandValue(parameters[val]) + } + } + + return nil +} diff --git a/ssm/expand_test.go b/ssm/expand_test.go new file mode 100644 index 0000000..c68b657 --- /dev/null +++ b/ssm/expand_test.go @@ -0,0 +1,64 @@ +package ssm + +import "testing" + +func TestExpandPresentNotPresent(t *testing.T) { + parameters := make(map[string]string) + + if err := expandParameters(parameters, false, []string{"test"}); err == nil { + t.Errorf("expected error when supplying var which is not present in params, but got nil") + } + + // set a value + parameters["test"] = "test value" + + if err := expandParameters(parameters, false, []string{"test"}); err != nil { + t.Errorf("expected no error, but got: %s", err) + } +} + +func TestExpandSelectiveExpansions(t *testing.T) { + t.Setenv("ENVIRONMENT", "teststaging") + + parameters := map[string]string{ + "DATABASE_NAME": "DB_$ENVIRONMENT", // should be expanded + "SOME_SECRET": "abc$abc", // should not be expanded + } + + // don't want to expand all here, just specific vars + if err := expandParameters(parameters, false, []string{"DATABASE_NAME"}); err != nil { + t.Errorf("expected no error, but got: %s", err) + } + + if parameters["DATABASE_NAME"] != "DB_teststaging" { + t.Errorf("DATABASE_NAME should be expanded to 'DB_teststaging', but got '%s'", parameters["DATABASE_NAME"]) + } + if parameters["SOME_SECRET"] != "abc$abc" { + t.Errorf("SOME_SECRET should not be expanded and be 'abc$abc', but got '%s'", parameters["SOME_SECRET"]) + } +} +func TestExpandExpansions(t *testing.T) { + t.Setenv("ENVIRONMENT", "teststaging") + t.Setenv("abc", "def") + + parameters := map[string]string{ + "DATABASE_NAME": "DB_$ENVIRONMENT", // should be expanded + "SOME_SECRET": "abc$abc", // should not be expanded + } + want := map[string]string{ + "DATABASE_NAME": "DB_teststaging", + "SOME_SECRET": "abcdef", + } + + // want to expand all + if err := expandParameters(parameters, true, []string{}); err != nil { + t.Errorf("expected no error, but got: %s", err) + } + + for key := range want { + if parameters[key] != want[key] { + t.Errorf("%s should be expanded to '%s', but got '%s'", key, want[key], parameters[key]) + } + } + +} diff --git a/ssm/parameters.go b/ssm/parameters.go index d1e16ad..19ea3bf 100644 --- a/ssm/parameters.go +++ b/ssm/parameters.go @@ -230,16 +230,18 @@ func getAllParameters(names, paths, plainNames, plainPaths []string, strict, rec } // GetParameters returns all parameters by path/names, with optional env vars expansion -func GetParameters(names, paths, plainNames, plainPaths []string, transformationsList []transformations.Transformation, expand, strict, recursive bool) (parameters map[string]string, err error) { +func GetParameters(names, paths, plainNames, plainPaths []string, transformationsList []transformations.Transformation, expand, strict, recursive, expandNames, expandPaths bool, expandValues []string) (parameters map[string]string, err error) { localNames := names localPaths := paths localPlainNames := plainNames localPlainPaths := plainPaths - if expand { + if expand || expandNames { localNames = ExpandArgs(names) - localPaths = ExpandArgs(paths) localPlainNames = ExpandArgs(plainNames) + } + if expand || expandPaths { + localPaths = ExpandArgs(paths) localPlainPaths = ExpandArgs(plainPaths) } allParameters, err := getAllParameters(localNames, localPaths, localPlainNames, localPlainPaths, strict, recursive) @@ -253,11 +255,11 @@ func GetParameters(names, paths, plainNames, plainPaths []string, transformation log.WithError(err).Fatal("Can't merge maps") } } - for key, value := range parameters { - if expand { - parameters[key] = ExpandValue(value) - } + + if err := expandParameters(parameters, expand, expandValues); err != nil { + log.WithError(err).Fatal("Can't expand vars") } + for _, transformation := range transformationsList { parameters, err = transformation.Transform(parameters) if err != nil { diff --git a/ssm/util.go b/ssm/util.go index fa808f8..39ecc10 100644 --- a/ssm/util.go +++ b/ssm/util.go @@ -2,9 +2,6 @@ package ssm import ( "os" - "strings" - - "github.com/buildkite/interpolate" ) var env Env @@ -25,26 +22,6 @@ func stringSliceDifference(a, b []string) []string { return ab } -// ExpandArgs expands arguments using env vars -func ExpandArgs(args []string) []string { - var expanded []string - for _, arg := range args { - arg = ExpandValue(arg) - expanded = append(expanded, arg) - } - return expanded -} - -// ExpandValue interpolates values using env vars -func ExpandValue(val string) string { - e, err := interpolate.Interpolate(env, val) - if err == nil { - return strings.TrimSpace(string(e)) - } - return val - -} - // Env just adapts os.LookupEnv to this interface type Env struct{}