Implement demo version of config grouping (#284)

* Implement basic config grouping command * Add better description and proto field comments * Add prompts for namespace and config and fake suggestions * Disable compilation outputs from printing to stdout * Use sort instead of slices package for go 1.20 * Fix lint
lekkodev · Jan 16, 2024 · f3a0da1 · f3a0da1
1 parent 63653ea
commit f3a0da1
Show file tree

Hide file tree

Showing 4 changed files with 354 additions and 0 deletions.
diff --git a/cmd/lekko/feature.go b/cmd/lekko/feature.go
@@ -18,19 +18,29 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"io"
 	"os"
+	"os/exec"
+	"path"
+	"sort"
 	"strings"
 	"text/tabwriter"
+	"time"
 
+	featurev1beta1 "buf.build/gen/go/lekkodev/cli/protocolbuffers/go/lekko/feature/v1beta1"
 	"github.com/AlecAivazis/survey/v2"
+	"github.com/briandowns/spinner"
+	"github.com/iancoleman/strcase"
 	"github.com/lekkodev/cli/pkg/feature"
 	"github.com/lekkodev/cli/pkg/logging"
 	"github.com/lekkodev/cli/pkg/metadata"
 	"github.com/lekkodev/cli/pkg/repo"
 	"github.com/lekkodev/cli/pkg/secrets"
+	"github.com/lekkodev/cli/pkg/star/static"
 	"github.com/lekkodev/go-sdk/pkg/eval"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
+	"google.golang.org/protobuf/reflect/protoreflect"
 	"google.golang.org/protobuf/types/known/structpb"
 )
 
@@ -45,6 +55,7 @@ func featureCmd() *cobra.Command {
 		featureAdd(),
 		featureRemove(),
 		featureEval(),
+		configGroup(),
 	)
 	return cmd
 }
@@ -299,6 +310,257 @@ func featureEval() *cobra.Command {
 	return cmd
 }
 
+func configGroup() *cobra.Command {
+	var ns, protoPkg, outName, description string
+	var configNames []string
+	cmd := &cobra.Command{
+		Use:   "group",
+		Short: "group multiple configs into 1 config",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			wd, err := os.Getwd()
+			if err != nil {
+				return err
+			}
+			r, err := repo.NewLocal(wd, secrets.NewSecretsOrFail())
+			if err != nil {
+				return err
+			}
+			// Don't output for compilations
+			// Downside: for unhappy path, compile errors will be less obvious
+			r.ConfigureLogger(&repo.LoggingConfiguration{
+				Writer: io.Discard,
+			})
+			ctx := cmd.Context()
+			// Take namespace input if necessary
+			if ns == "" {
+				nss, err := r.ListNamespaces(ctx)
+				if err != nil {
+					return errors.Wrap(err, "list namespaces")
+				}
+				var options []string
+				for _, ns := range nss {
+					options = append(options, ns.Name)
+				}
+				if err := survey.AskOne(&survey.Select{
+					Message: "Namespace:",
+					Options: options,
+				}, &ns); err != nil {
+					return errors.Wrap(err, "prompt for namespace")
+				}
+			}
+			allNsfs, err := getNamespaceFeatures(ctx, r, ns, "")
+			if err != nil {
+				return err
+			}
+			// Take configs input if necessary
+			if len(configNames) == 0 {
+				var options []string
+				for _, nsf := range allNsfs {
+					options = append(options, nsf.featureName)
+				}
+				sort.Strings(options)
+				// NOTE: Currently this doesn't respect selection order
+				// so if someone wants a specific order they have to pass the flag explicitly
+				if err := survey.AskOne(&survey.MultiSelect{
+					Message: "Configs to group:",
+					Options: options,
+				}, &configNames, survey.WithValidator(survey.MinItems(2))); err != nil {
+					return errors.Wrap(err, "prompt for configs")
+				}
+			}
+			if len(configNames) < 2 {
+				return errors.New("at least 2 configs must be specified for grouping")
+			}
+			cMap := make(map[string]struct{})
+			for _, nsf := range allNsfs {
+				cMap[nsf.featureName] = struct{}{}
+			}
+			// Check all specified configs exist
+			for _, c := range configNames {
+				if _, ok := cMap[c]; !ok {
+					return errors.Errorf("config %s/%s not found", ns, c)
+				}
+			}
+			// Compile to check for healthy state and get compiled objects (for type info)
+			rootMD, nsMDs, err := r.ParseMetadata(ctx)
+			if err != nil {
+				return errors.Wrap(err, "parse metadata")
+			}
+			// Can't support grouping if using external proto types
+			if rootMD.UseExternalTypes {
+				return errors.New("grouping is not supported with external protobuf types")
+			}
+			registry, err := r.ReBuildDynamicTypeRegistry(ctx, rootMD.ProtoDirectory, rootMD.UseExternalTypes)
+			if err != nil {
+				return errors.Wrap(err, "rebuild type registry")
+			}
+			compileResults, err := r.Compile(ctx, &repo.CompileRequest{
+				Registry:        registry,
+				NamespaceFilter: ns,
+			})
+			if err != nil {
+				return errors.Wrap(err, "pre-group compile")
+			}
+			var compiledList []*feature.Feature
+			compiledMap := make(map[string]*feature.Feature)
+			for _, res := range compileResults {
+				compiledMap[res.FeatureName] = res.CompiledFeature.Feature
+			}
+			for _, c := range configNames {
+				compiledList = append(compiledList, compiledMap[c])
+			}
+			// Prompt for grouped name if necessary, using suggestion engine
+			if outName == "" {
+				// tab for suggestions?
+				if err := survey.AskOne(&survey.Input{
+					Message: "Grouped config name:",
+					Suggest: func(_ string) []string {
+						s := spinner.New(spinner.CharSets[14], 100*time.Millisecond)
+						s.Suffix = " Generating AI suggestions..."
+						s.Start()
+						suggestions := feature.SuggestGroupedNames(compiledList...)
+						s.Stop()
+						return suggestions
+					},
+				}, &outName); err != nil {
+					return errors.Wrap(err, "prompt for grouped name")
+				}
+			}
+			// Generate new proto message def string based on config types
+			// TODO: handle name collisions
+			protoMsgName := strcase.ToCamel(outName)
+			pdf := feature.NewProtoDefBuilder(protoMsgName)
+			protoFieldNames := make([]string, len(configNames))
+			for i, c := range configNames {
+				pt, err := pdf.ToProtoTypeName(compiledMap[c].FeatureType)
+				if err != nil {
+					return err
+				}
+				// Use original description as comment in generated proto
+				protoFieldNames[i] = pdf.AddField(c, pt, compiledMap[c].Description)
+			}
+			// Update proto file
+			protoPath := path.Join(rootMD.ProtoDirectory, strings.ReplaceAll(protoPkg, ".", "/"), "gen.proto")
+			if _, err := os.Stat(protoPath); errors.Is(err, os.ErrNotExist) {
+				if err := os.MkdirAll(path.Dir(protoPath), 0775); err != nil {
+					return errors.Wrap(err, "create destination proto file")
+				}
+				pf, err := os.Create(protoPath)
+				if err != nil {
+					return errors.Wrap(err, "create destination proto file")
+				}
+				defer pf.Close()
+				// Write proto file preamble
+				if _, err := pf.WriteString(fmt.Sprintf("syntax = \"proto3\";\n\npackage %s;\n\n", protoPkg)); err != nil {
+					return errors.Wrap(err, "write preamble to destination proto file")
+				}
+			}
+			pf, err := os.OpenFile(protoPath, os.O_APPEND|os.O_WRONLY, 0644)
+			if err != nil {
+				return errors.Wrap(err, "open destination proto file for write")
+			}
+			defer pf.Close()
+			if _, err := pf.WriteString(pdf.Build()); err != nil {
+				return errors.Wrap(err, "write to destination proto file")
+			}
+			// Format & rebuild registry for new type
+			formatCmd := exec.Command("buf", "format", protoPath, "-w")
+			if err := formatCmd.Run(); err != nil {
+				return errors.Wrap(err, "buf format")
+			}
+			registry, err = r.ReBuildDynamicTypeRegistry(ctx, rootMD.ProtoDirectory, rootMD.UseExternalTypes)
+			if err != nil {
+				return errors.Wrap(err, "rebuild type registry")
+			}
+			// Create new config using generated proto type
+			protoFullName := strings.Join([]string{protoPkg, protoMsgName}, ".")
+			_, err = r.AddFeature(ctx, ns, outName, eval.ConfigTypeProto, protoFullName)
+			if err != nil {
+				return errors.Wrap(err, "add new config")
+			}
+			compileResults, err = r.Compile(ctx, &repo.CompileRequest{
+				Registry:        registry,
+				NamespaceFilter: ns,
+				FeatureFilter:   outName,
+			})
+			if err != nil {
+				return errors.Wrap(err, "add and compile new config")
+			}
+			newF := compileResults[0].CompiledFeature.Feature
+			// Update default value based on original configs' default values
+			// TODO: Handle overrides, handling precedence based on arg order
+			mt, err := registry.FindMessageByName(protoreflect.FullName(protoFullName))
+			if err != nil {
+				return errors.Wrap(err, "find message")
+			}
+			defaultValue := mt.New()
+			for i, c := range configNames {
+				orig := compiledMap[c]
+				defaultValue.Set(mt.Descriptor().Fields().ByName(protoreflect.Name(protoFieldNames[i])), protoreflect.ValueOf(orig.Value))
+			}
+			newF.Value = defaultValue
+			sf, err := r.Parse(ctx, ns, outName, registry)
+			if err != nil {
+				return errors.Wrap(err, "parse generated config")
+			}
+			newFProto, err := newF.ToProto()
+			if err != nil {
+				return errors.Wrap(err, "convert before mutation")
+			}
+			newFF, err := r.GetFeatureFile(ctx, ns, outName)
+			if err != nil {
+				return errors.Wrap(err, "get new config file")
+			}
+			newFBytes, err := os.ReadFile(path.Join(ns, newFF.StarlarkFileName))
+			if err != nil {
+				return errors.Wrap(err, "read new config starlark")
+			}
+			// TODO: description suggestion
+			if description == "" {
+				description = fmt.Sprintf("Grouped from %s", strings.Join(configNames, ", "))
+			}
+			newFProto.Description = description
+			newFBytes, err = static.NewWalker(newFF.StarlarkFileName, newFBytes, registry, feature.NamespaceVersion(nsMDs[ns].Version)).Mutate(&featurev1beta1.StaticFeature{
+				Key:  newFProto.Key,
+				Type: newFProto.Type,
+				Feature: &featurev1beta1.FeatureStruct{
+					Description: newFProto.Description,
+				},
+				FeatureOld: newFProto,
+				Imports:    sf.Imports,
+			})
+			if err != nil {
+				return errors.Wrap(err, "mutate new config")
+			}
+			if err := r.WriteFile(path.Join(ns, newFF.StarlarkFileName), newFBytes, 0600); err != nil {
+				return errors.Wrap(err, "write after mutation")
+			}
+			// Remove old configs
+			for _, c := range configNames {
+				if err := r.RemoveFeature(ctx, ns, c); err != nil {
+					return errors.Wrapf(err, "remove config %s/%s", ns, c)
+				}
+			}
+			// Final compile
+			_, err = r.Compile(ctx, &repo.CompileRequest{
+				Registry:        registry,
+				NamespaceFilter: ns,
+			})
+			if err != nil {
+				return errors.Wrap(err, "compile after mutation")
+			}
+			return nil
+		},
+	}
+	// This might not be the cleanest CLI design, but not sure how to do it cleaner
+	cmd.Flags().StringVarP(&outName, "out", "o", "", "name of grouped config")
+	cmd.Flags().StringVarP(&ns, "namespace", "n", "", "namespace of configs to group together")
+	cmd.Flags().StringSliceVarP(&configNames, "configs", "c", []string{}, "comma-separated names of configs to group together")
+	cmd.Flags().StringVarP(&protoPkg, "proto-pkg", "p", "default.config.v1beta1", "package for generated protobuf type(s)")
+	cmd.Flags().StringVarP(&description, "description", "d", "", "description for the grouped config")
+	return cmd
+}
+
 func namespaceCmd() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "ns",

diff --git a/go.mod b/go.mod
@@ -7,11 +7,13 @@ require (
 	buf.build/gen/go/lekkodev/cli/protocolbuffers/go v1.31.0-20231020162356-e763402ec965.1
 	github.com/AlecAivazis/survey/v2 v2.3.6
 	github.com/bazelbuild/buildtools v0.0.0-20220907133145-b9bfff5d7f91
+	github.com/briandowns/spinner v1.23.0
 	github.com/bufbuild/connect-go v1.10.0
 	github.com/cli/browser v1.0.0
 	github.com/go-git/go-billy/v5 v5.4.1
 	github.com/go-git/go-git/v5 v5.8.0
 	github.com/google/go-github/v52 v52.0.0
+	github.com/iancoleman/strcase v0.3.0
 	github.com/lekkodev/go-sdk v0.2.6-0.20230830172236-f072eb8bf64e
 	github.com/lekkodev/rules v1.5.3-0.20230724195144-d0ed93c3e218
 	github.com/migueleliasweb/go-github-mock v0.0.16
@@ -41,6 +43,7 @@ require (
 	github.com/cloudflare/circl v1.3.3 // indirect
 	github.com/emicklei/go-restful/v3 v3.8.0 // indirect
 	github.com/emirpasic/gods v1.18.1 // indirect
+	github.com/fatih/color v1.7.0 // indirect
 	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
 	github.com/go-logr/logr v1.2.3 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect

diff --git a/go.sum b/go.sum
@@ -29,6 +29,8 @@ github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/bazelbuild/buildtools v0.0.0-20220907133145-b9bfff5d7f91 h1:7xSt0nPZ74liqR8jjBYmjentrQVrdQEhoW4/+4BrmoM=
 github.com/bazelbuild/buildtools v0.0.0-20220907133145-b9bfff5d7f91/go.mod h1:689QdV3hBP7Vo9dJMmzhoYIyo/9iMhEmHkJcnaPRCbo=
+github.com/briandowns/spinner v1.23.0 h1:alDF2guRWqa/FOZZYWjlMIx2L6H0wyewPxo/CH4Pt2A=
+github.com/briandowns/spinner v1.23.0/go.mod h1:rPG4gmXeN3wQV/TsAY4w8lPdIM6RX3yqeBQJSrbXjuE=
 github.com/bufbuild/connect-go v1.10.0 h1:QAJ3G9A1OYQW2Jbk3DeoJbkCxuKArrvZgDt47mjdTbg=
 github.com/bufbuild/connect-go v1.10.0/go.mod h1:CAIePUgkDR5pAFaylSMtNK45ANQjp9JvpluG20rhpV8=
 github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
@@ -64,6 +66,8 @@ github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc
 github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
+github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY=
 github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI=
 github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic=
@@ -131,6 +135,8 @@ github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
 github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec h1:qv2VnGeEQHchGaZ/u7lxST/RaJw+cv273q79D81Xbog=
 github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec/go.mod h1:Q48J4R4DvxnHolD5P8pOtXigYlRuPLGl6moFx3ulM68=
+github.com/iancoleman/strcase v0.3.0 h1:nTXanmYxhfFAMjZL34Ov6gkzEsSJZ5DbhxWjvSASxEI=
+github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
 github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM=
 github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=