From a72c40b31dea5b13c24be64c7b7279c993333ca9 Mon Sep 17 00:00:00 2001 From: David Kang Date: Fri, 5 Jan 2024 11:29:37 -0800 Subject: [PATCH 1/4] Fix config eval command version support and formatting (#276) --- cmd/lekko/feature.go | 2 +- pkg/encoding/encoding.go | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/lekko/feature.go b/cmd/lekko/feature.go index afd1ace9..41c0cc26 100644 --- a/cmd/lekko/feature.go +++ b/cmd/lekko/feature.go @@ -281,7 +281,7 @@ func featureEval() *cobra.Command { } fmt.Fprintf(os.Stderr, "[%s] ", fType) - fmt.Printf("%s", res) + fmt.Printf("%v", res) fmt.Println() if verbose { fmt.Fprintf(os.Stderr, "[path] %v\n", path) diff --git a/pkg/encoding/encoding.go b/pkg/encoding/encoding.go index 17c0240a..bbb27523 100644 --- a/pkg/encoding/encoding.go +++ b/pkg/encoding/encoding.go @@ -40,6 +40,8 @@ func ParseFeature(ctx context.Context, rootPath string, featureFile feature.Feat case feature.NamespaceVersionV1Beta5.String(): fallthrough case feature.NamespaceVersionV1Beta6.String(): + fallthrough + case feature.NamespaceVersionV1Beta7.String(): var f featurev1beta1.Feature contents, err := provider.GetFileContents(ctx, filepath.Join(rootPath, nsMD.Name, featureFile.CompiledProtoBinFileName)) if err != nil { From 51927cf736b2764f6215c10fada51a88d7a5e747 Mon Sep 17 00:00:00 2001 From: David Kang Date: Tue, 9 Jan 2024 13:20:32 -0800 Subject: [PATCH 2/4] Add metadata to BFF Feature proto (#274) * Add metadata to BFF Feature proto * Move comment as per feedback --- proto/lekko/bff/v1beta1/bff.proto | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/proto/lekko/bff/v1beta1/bff.proto b/proto/lekko/bff/v1beta1/bff.proto index a8ec1f89..df751801 100644 --- a/proto/lekko/bff/v1beta1/bff.proto +++ b/proto/lekko/bff/v1beta1/bff.proto @@ -468,6 +468,15 @@ message GetFeatureHistoryResponse { repeated FeatureHistoryItem history = 1; } +// The configs will have metadata fields populated based on the latest rolled +// out version of the repo. +// If a config was newly created since, created_at will not be set. +// If a config existed already but was updated, last_updated_at will not be set. +message ConfigMetadata { + google.protobuf.Timestamp created_at = 1; + google.protobuf.Timestamp last_updated_at = 2; +} + message Feature { string name = 1; string namespace_name = 2; @@ -479,6 +488,7 @@ message Feature { // the blob sha of the proto bin file according to git string sha = 6; lekko.feature.v1beta1.StaticFeature static_feature_new = 7; + ConfigMetadata metadata = 8; } message FeatureHistoryItem { From cf052745415b9afb4777d356f278fd4027c5d44f Mon Sep 17 00:00:00 2001 From: David Kang Date: Tue, 9 Jan 2024 14:38:10 -0800 Subject: [PATCH 3/4] Introduce GitHub GraphQL client (#272) --- go.mod | 2 ++ go.sum | 4 ++++ pkg/gh/github.go | 9 +++++---- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 32dee689..4eb68c3d 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/migueleliasweb/go-github-mock v0.0.16 github.com/mitchellh/go-homedir v1.1.0 github.com/olekukonko/tablewriter v0.0.5 + github.com/shurcooL/githubv4 v0.0.0-20231126234147-1cffa1f02456 github.com/spf13/cobra v1.5.0 github.com/stretchr/testify v1.8.0 github.com/whilp/git-urls v1.0.0 @@ -69,6 +70,7 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/sergi/go-diff v1.1.0 // indirect + github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466 // indirect github.com/skeema/knownhosts v1.1.1 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect golang.org/x/crypto v0.9.0 // indirect diff --git a/go.sum b/go.sum index 25a79465..450644aa 100644 --- a/go.sum +++ b/go.sum @@ -200,6 +200,10 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/shurcooL/githubv4 v0.0.0-20231126234147-1cffa1f02456 h1:6dExqsYngGEiixqa1vmtlUd+zbyISilg0Cf3GWVdeYM= +github.com/shurcooL/githubv4 v0.0.0-20231126234147-1cffa1f02456/go.mod h1:zqMwyHmnN/eDOZOdiTohqIUKUrTFX62PNlu7IJdu0q8= +github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466 h1:17JxqqJY66GmZVHkmAsGEkcIu0oCe3AM420QDgGwZx0= +github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466/go.mod h1:9dIRpgIY7hVhoqfe0/FcYp0bpInZaT7dc3BYOprrIUE= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/skeema/knownhosts v1.1.1 h1:MTk78x9FPgDFVFkDLTrsnnfCJl7g1C/nnKvePgrIngE= github.com/skeema/knownhosts v1.1.1/go.mod h1:g4fPeYpque7P0xefxtGzV81ihjC8sX2IqpAoNkjxbMo= diff --git a/pkg/gh/github.go b/pkg/gh/github.go index f938b041..a77a0432 100644 --- a/pkg/gh/github.go +++ b/pkg/gh/github.go @@ -24,6 +24,7 @@ import ( "github.com/go-git/go-git/v5" "github.com/google/go-github/v52/github" "github.com/pkg/errors" + "github.com/shurcooL/githubv4" "golang.org/x/oauth2" ) @@ -37,20 +38,20 @@ const ( // relevant to lekko. type GithubClient struct { *github.Client + Graphql *githubv4.Client } func NewGithubClient(h *http.Client) *GithubClient { return &GithubClient{ - Client: github.NewClient(h), + Client: github.NewClient(h), + Graphql: githubv4.NewClient(h), } } func NewGithubClientFromToken(ctx context.Context, token string) *GithubClient { ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token}) tc := oauth2.NewClient(ctx, ts) - return &GithubClient{ - Client: github.NewClient(tc), - } + return NewGithubClient(tc) } func (gc *GithubClient) GetUser(ctx context.Context) (*github.User, error) { From e695609b9c88948427d4f3009c36b46858827546 Mon Sep 17 00:00:00 2001 From: lekko-jonathan <150070021+lekko-jonathan@users.noreply.github.com> Date: Fri, 12 Jan 2024 11:36:32 -0800 Subject: [PATCH 4/4] Add the ability to declare static context keys per namespace (#279) * Add the ability to declare static context keys per namespace Bask in eternal beauty of the protobuf API * Remove pointless logging * address my own code feedback --------- Co-authored-by: Konrad Niemiec --- pkg/metadata/metadata.go | 5 +- pkg/repo/feature.go | 155 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 151 insertions(+), 9 deletions(-) diff --git a/pkg/metadata/metadata.go b/pkg/metadata/metadata.go index 62b3259c..5a7bfaf2 100644 --- a/pkg/metadata/metadata.go +++ b/pkg/metadata/metadata.go @@ -50,8 +50,9 @@ type RootConfigRepoMetadata struct { type NamespaceConfigRepoMetadata struct { // This version refers to the version of the configuration in the repo itself. // TODO we should move this to a separate version number. - Version string `json:"version,omitempty" yaml:"version,omitempty"` - Name string `json:"name,omitempty" yaml:"name,omitempty"` + Version string `json:"version,omitempty" yaml:"version,omitempty"` + Name string `json:"name,omitempty" yaml:"name,omitempty"` + ContextProto string `json:"contextProto,omitempty" yaml:"contextProto,omitempty"` } const DefaultRootConfigRepoMetadataFileName = "lekko.root.yaml" diff --git a/pkg/repo/feature.go b/pkg/repo/feature.go index c774d6f9..c873b0e1 100644 --- a/pkg/repo/feature.go +++ b/pkg/repo/feature.go @@ -25,6 +25,7 @@ import ( "sync" featurev1beta1 "buf.build/gen/go/lekkodev/cli/protocolbuffers/go/lekko/feature/v1beta1" + rulesv1beta3 "buf.build/gen/go/lekkodev/cli/protocolbuffers/go/lekko/rules/v1beta3" "github.com/go-git/go-git/v5/plumbing" "github.com/lekkodev/cli/pkg/encoding" "github.com/lekkodev/cli/pkg/feature" @@ -39,6 +40,7 @@ import ( "google.golang.org/protobuf/reflect/protoregistry" "google.golang.org/protobuf/types/descriptorpb" "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/structpb" ) // Provides functionality needed for accessing and making changes to Lekko configuration. @@ -67,7 +69,110 @@ type ConfigurationStore interface { RestoreWorkingDirectory(hash string) error } -func (r *repository) CompileFeature(ctx context.Context, registry *protoregistry.Types, namespace, featureName string, nv feature.NamespaceVersion) (*feature.CompiledFeature, error) { +func checkRuleFitsContextType(rule *rulesv1beta3.Rule, contextType protoreflect.MessageType) error { + var err error + switch r := rule.Rule.(type) { + case *rulesv1beta3.Rule_BoolConst: + return nil + case *rulesv1beta3.Rule_Not: + return checkRuleFitsContextType(r.Not, contextType) + case *rulesv1beta3.Rule_LogicalExpression: + for _, lr := range r.LogicalExpression.GetRules() { + err = checkRuleFitsContextType(lr, contextType) + if err != nil { + return err + } + } + case *rulesv1beta3.Rule_Atom: + field := contextType.Descriptor().Fields().ByName(protoreflect.Name(r.Atom.GetContextKey())) + if field == nil { + return errors.Errorf("`%s` field not found in context type", r.Atom.GetContextKey()) + } + fieldType := field.Kind().String() + + switch r.Atom.ComparisonOperator { + case rulesv1beta3.ComparisonOperator_COMPARISON_OPERATOR_PRESENT: + return nil + case + rulesv1beta3.ComparisonOperator_COMPARISON_OPERATOR_EQUALS, + rulesv1beta3.ComparisonOperator_COMPARISON_OPERATOR_NOT_EQUALS: + switch r.Atom.GetComparisonValue().Kind.(type) { + case *structpb.Value_BoolValue: + if fieldType != "string" { + return errors.Errorf("%s field has invalid type", r.Atom.GetContextKey()) + } + + case *structpb.Value_NumberValue: + if fieldType != "int64" && fieldType != "double" { + return errors.Errorf("%s field has invalid type", r.Atom.GetContextKey()) + } + case *structpb.Value_StringValue: + if fieldType != "string" { + return errors.Errorf("%s field has invalid type", r.Atom.GetContextKey()) + } + default: + panic("This should never happen") + } + case + rulesv1beta3.ComparisonOperator_COMPARISON_OPERATOR_LESS_THAN, + rulesv1beta3.ComparisonOperator_COMPARISON_OPERATOR_LESS_THAN_OR_EQUALS, + rulesv1beta3.ComparisonOperator_COMPARISON_OPERATOR_GREATER_THAN, + rulesv1beta3.ComparisonOperator_COMPARISON_OPERATOR_GREATER_THAN_OR_EQUALS: + if fieldType != "int64" && fieldType != "double" { + return errors.Errorf("%s field has invalid type", r.Atom.GetContextKey()) + } + return nil + case + rulesv1beta3.ComparisonOperator_COMPARISON_OPERATOR_STARTS_WITH, + rulesv1beta3.ComparisonOperator_COMPARISON_OPERATOR_ENDS_WITH, + rulesv1beta3.ComparisonOperator_COMPARISON_OPERATOR_CONTAINS: + if fieldType != "string" { + return errors.Errorf("%s field has invalid type", r.Atom.GetContextKey()) + } + return nil + case rulesv1beta3.ComparisonOperator_COMPARISON_OPERATOR_CONTAINED_WITHIN: + listRuleVal, ok := r.Atom.GetComparisonValue().Kind.(*structpb.Value_ListValue) + if !ok { + panic("This should never happen") + } + for _, elem := range listRuleVal.ListValue.Values { + switch elem.Kind.(type) { + case *structpb.Value_BoolValue: + if fieldType != "string" { + return errors.Errorf("%s field has invalid type", r.Atom.GetContextKey()) + } + + case *structpb.Value_NumberValue: + if fieldType != "int64" && fieldType != "double" { + return errors.Errorf("%s field has invalid type", r.Atom.GetContextKey()) + } + case *structpb.Value_StringValue: + if fieldType != "string" { + return errors.Errorf("%s field has invalid type", r.Atom.GetContextKey()) + } + default: + panic("This should never happen") + } + } + return nil + } + return nil + case *rulesv1beta3.Rule_CallExpression: + switch f := r.CallExpression.Function.(type) { + case *rulesv1beta3.CallExpression_Bucket_: + contextKey := f.Bucket.ContextKey + if contextType.Descriptor().Fields().ByName(protoreflect.Name(contextKey)) == nil { + return errors.Errorf("%s field not found in context type", contextKey) + } + return nil + } + default: + panic("This should never happen") + } + return nil +} + +func (r *repository) CompileFeature(ctx context.Context, registry *protoregistry.Types, nsContextTypes map[string]protoreflect.MessageType, namespace, featureName string, nv feature.NamespaceVersion) (*feature.CompiledFeature, error) { if !isValidName(namespace) { return nil, errors.Errorf("invalid name '%s'", namespace) } @@ -92,6 +197,14 @@ func (r *repository) CompileFeature(ctx context.Context, registry *protoregistry if err != nil { return nil, errors.Wrap(err, "compile") } + if nsContextTypes[namespace] != nil { + for _, override := range f.Feature.Overrides { + err = checkRuleFitsContextType(override.RuleASTV3, nsContextTypes[namespace]) + if err != nil { + return nil, err + } + } + } return f, nil } @@ -303,6 +416,38 @@ func (r *repository) Compile(ctx context.Context, req *CompileRequest) ([]*Featu if err := req.Validate(); err != nil { return nil, errors.Wrap(err, "validate request") } + + _, nsMDs, err := r.ParseMetadata(ctx) + if err != nil { + return nil, errors.Wrap(err, "parse metadata") + } + registry, err := r.registry(ctx, req.Registry) + if err != nil { + return nil, errors.Wrap(err, "registry") + } + nsContextTypes := make(map[string]protoreflect.MessageType) + for ns, nsMd := range nsMDs { + if nsMd.ContextProto != "" { + ct, err := registry.FindMessageByName(protoreflect.FullName(nsMd.ContextProto)) + if err != nil { + return nil, err + } + for i := 0; i < ct.Descriptor().Fields().Len(); i++ { + f := ct.Descriptor().Fields().Get(i) + switch f.Kind().String() { + case + "bool", + "int64", + "double", + "string": + default: + return nil, errors.Errorf("proto message cannot be used as a context message because type: %v of key: %s is not allowed", f.Kind(), f.Name()) + } + } + nsContextTypes[ns] = ct + } + } + // Step 1: collect. Find all features vffs, numNamespaces, err := r.findVersionedFeatureFiles(ctx, req.NamespaceFilter, req.FeatureFilter, req.Verify) if err != nil { @@ -326,10 +471,6 @@ func (r *repository) Compile(ctx context.Context, req *CompileRequest) ([]*Featu } r.Logf("Found %d configs across %d namespaces\n", len(results), numNamespaces) r.Logf("Compiling...\n") - registry, err := r.registry(ctx, req.Registry) - if err != nil { - return nil, errors.Wrap(err, "registry") - } concurrency := 50 if len(results) < 50 { concurrency = len(results) @@ -344,7 +485,7 @@ func (r *repository) Compile(ctx context.Context, req *CompileRequest) ([]*Featu nsNameToSegments := make(map[string]map[string]string) for _, fcr := range results { if fcr.FeatureName == "segments" { - cf, err := r.CompileFeature(ctx, registry, fcr.NamespaceName, fcr.FeatureName, fcr.NamespaceVersion) + cf, err := r.CompileFeature(ctx, registry, nsContextTypes, fcr.NamespaceName, fcr.FeatureName, fcr.NamespaceVersion) if err != nil { fcr.CompilationError = err } @@ -385,7 +526,7 @@ func (r *repository) Compile(ctx context.Context, req *CompileRequest) ([]*Featu } fcr.FormattingDiffExists = fmtDiffExists // compile feature - cf, err := r.CompileFeature(ctx, registry, fcr.NamespaceName, fcr.FeatureName, fcr.NamespaceVersion) + cf, err := r.CompileFeature(ctx, registry, nsContextTypes, fcr.NamespaceName, fcr.FeatureName, fcr.NamespaceVersion) fcr.CompiledFeature = cf fcr.CompilationError = err }