Hook up new TS repoless sync and update sync/bisync commands

Author: DavidSGK

cmd/lekko/bisync.go

@@ -59,7 +59,7 @@ func bisync(ctx context.Context, project *native.Project, lekkoPath, repoPath st
 	case native.LangGo:
 		_ = try.To1(sync.BisyncGo(ctx, lekkoPath, lekkoPath, repoPath))
 	case native.LangTypeScript:
-		try.To(sync.BisyncTS(lekkoPath, repoPath))
+		try.To(sync.BisyncTS(ctx, lekkoPath, repoPath))
 	default:
 		return errors.New("unsupported language")
 	}

cmd/lekko/gen.go

@@ -82,7 +82,7 @@ func genGoCmd() *cobra.Command {
 	var initMode bool
 	cmd := &cobra.Command{
 		Use:   "go",
-		Short: "generate Go library code from configs",
+		Short: "generate Go library code from lekkos",
 		RunE: func(cmd *cobra.Command, args []string) error {
 			nlProject := try.To1(native.DetectNativeLang(""))
 			if nlProject.Language != native.LangGo {
@@ -104,11 +104,11 @@ func genTSCmd() *cobra.Command {
 	var lekkoPath string
 	cmd := &cobra.Command{
 		Use:   "ts",
-		Short: "generate typescript library code from configs",
+		Short: "generate TypeScript library code from lekkos",
 		RunE: func(cmd *cobra.Command, args []string) error {
 			nlProject := try.To1(native.DetectNativeLang(""))
 			if nlProject.Language != native.LangTypeScript {
-				return errors.Errorf("not a Go project, detected %v instead", nlProject.Language)
+				return errors.Errorf("not a TypeScript project, detected %v instead", nlProject.Language)
 			}
 			return genNative(cmd.Context(), nlProject, lekkoPath, repoPath, ns, false)
 		},

cmd/lekko/repo.go

@@ -490,6 +490,7 @@ func pullCmd() *cobra.Command {
 
 			switch nlProject.Language {
 			case native.LangTypeScript:
+				// TODO: Fix pull for TS, this command no longer exists and we have repoless sync/gen now
 				tsPullCmd := exec.Command("npx", "lekko-repo-pull", "--lekko-dir", lekkoPath)
 				output, err := tsPullCmd.CombinedOutput()
 				fmt.Println(string(output))

cmd/lekko/sync.go

@@ -54,6 +54,7 @@ func syncCmd() *cobra.Command {
 		Short: "sync code to config",
 	}
 	cmd.AddCommand(syncGoCmd())
+	cmd.AddCommand(syncTSCmd())
 	return cmd
 }
 
@@ -90,6 +91,44 @@ func syncGoCmd() *cobra.Command {
 	return cmd
 }
 
+func syncTSCmd() *cobra.Command {
+	var repoPath string
+	cmd := &cobra.Command{
+		Use:   "ts filepath1 filepath2 ...",
+		Short: "parse TypeScript files with lekkos to a local config repository",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			ctx := cmd.Context()
+			var err error
+			if len(repoPath) == 0 {
+				repoPath, err = repo.PrepareGithubRepo()
+				if err != nil {
+					return err
+				}
+			}
+			var repoContents *featurev1beta1.RepositoryContents
+			if len(args) > 0 {
+				repoContents, err = sync.SyncTSFiles(args...)
+			} else {
+				dot, doterr := dotlekko.ReadDotLekko("")
+				if doterr != nil {
+					return doterr
+				}
+				repoContents, err = sync.SyncTS(dot.LekkoPath)
+			}
+			if err != nil {
+				return err
+			}
+			err = sync.WriteContentsToLocalRepo(ctx, repoContents, repoPath)
+			if err != nil {
+				return err
+			}
+			return nil
+		},
+	}
+	cmd.Flags().StringVarP(&repoPath, "repo-path", "r", "", "path to config repository, will use autodetect if not set")
+	return cmd
+}
+
 func anyToLekkoAny(a *anypb.Any) *featurev1beta1.Any {
 	return &featurev1beta1.Any{
 		TypeUrl: a.GetTypeUrl(),

pkg/gen/gen.go

@@ -81,7 +81,7 @@ func GenNative(ctx context.Context, project *native.Project, lekkoPath, repoPath
 		}
 		for _, ns := range opts.Namespaces {
 			outFilename := filepath.Join(absLekkoPath, ns+project.Language.Ext())
-			try.To(genFormattedTS(ctx, repoPath, ns, outFilename))
+			try.To(GenFormattedTS(ctx, repoPath, ns, outFilename))
 		}
 		return nil
 	case native.LangGo:

pkg/gen/golang.go

@@ -324,7 +324,7 @@ func renderGoTemplate(templateBody string, fileName string, data any) (string, e
 }
 
 // Generates public and private function files for the namespace as well as the overall client file.
-// Writes outputs to the output paths specified in the
+// Writes outputs to the output paths specified in the construction args.
 // TODO: since generator takes in whole repo contents now, could generate for all/filtered namespaces
 func (g *goGenerator) Gen(ctx context.Context, namespaceName string) (err error) {
 	defer err2.Handle(&err)

pkg/gen/ts.go

@@ -138,7 +138,7 @@ func getTSParameters(d protoreflect.MessageDescriptor) string {
 }
 
 // Pipe `GenTS` to prettier
-func genFormattedTS(ctx context.Context, repoPath, ns, outFilename string) error {
+func GenFormattedTS(ctx context.Context, repoPath, ns, outFilename string) error {
 	prettierCmd := exec.Command("npx", "prettier", "--parser", "typescript")
 	stdinPipe, err := prettierCmd.StdinPipe()
 	if err != nil {

pkg/native/native.go

@@ -137,8 +137,8 @@ func NativeLangFromExt(filename string) (Language, error) {
 	return "", fmt.Errorf("unsupported file extension: %v", ext)
 }
 
-func (l *Language) Ext() string {
-	switch *l {
+func (l Language) Ext() string {
+	switch l {
 	case LangGo:
 		return ".go"
 	case LangTypeScript:

pkg/proto/proto.go

@@ -26,6 +26,7 @@ import (
 	"google.golang.org/protobuf/types/dynamicpb"
 	"google.golang.org/protobuf/types/known/anypb"
 	"google.golang.org/protobuf/types/known/durationpb"
+	"google.golang.org/protobuf/types/known/emptypb"
 	"google.golang.org/protobuf/types/known/structpb"
 	"google.golang.org/protobuf/types/known/wrapperspb"
 
@@ -172,3 +173,34 @@ func PrintMessageDescriptor(md protoreflect.MessageDescriptor, indentLevel int)
 	sb.WriteString(fmt.Sprintf("%s}\n", indent))
 	return sb.String(), nil
 }
+
+// A custom resolver for unmarshalling that will replace unresolvable Anys with emptypb.Empty messages
+type IgnoreAnyResolver struct{}
+
+func (r *IgnoreAnyResolver) FindMessageByName(message protoreflect.FullName) (protoreflect.MessageType, error) {
+	mt, err := protoregistry.GlobalTypes.FindMessageByName(message)
+	if err != nil {
+		return nil, errors.Wrap(err, "name?")
+	}
+	return mt, nil
+}
+
+func (r *IgnoreAnyResolver) FindMessageByURL(url string) (protoreflect.MessageType, error) {
+	mt, err := protoregistry.GlobalTypes.FindMessageByURL(url)
+	if errors.Is(err, protoregistry.NotFound) {
+		e := &emptypb.Empty{}
+		return e.ProtoReflect().Type(), nil
+	}
+	if err != nil {
+		return nil, errors.Wrap(err, "url?")
+	}
+	return mt, nil
+}
+
+func (r *IgnoreAnyResolver) FindExtensionByName(field protoreflect.FullName) (protoreflect.ExtensionType, error) {
+	return protoregistry.GlobalTypes.FindExtensionByName(field)
+}
+
+func (r *IgnoreAnyResolver) FindExtensionByNumber(message protoreflect.FullName, field protoreflect.FieldNumber) (protoreflect.ExtensionType, error) {
+	return protoregistry.GlobalTypes.FindExtensionByNumber(message, field)
+}

pkg/sync/golang.go

@@ -55,9 +55,7 @@ func BisyncGo(ctx context.Context, outputPath, lekkoPath, repoPath string) ([]st
 	if err != nil {
 		return nil, err
 	}
-
 	// Traverse target path, finding namespaces
-	// TODO: consider making this more efficient for batch gen/sync
 	files := make([]string, 0)
 	if err := filepath.WalkDir(lekkoPath, func(p string, d fs.DirEntry, err error) error {
 		if err != nil {
@@ -67,7 +65,7 @@ func BisyncGo(ctx context.Context, outputPath, lekkoPath, repoPath string) ([]st
 		if d.IsDir() && d.Name() == "proto" {
 			return filepath.SkipDir
 		}
-		// Sync and gen - only target <namespace>/<namespace>.go files
+		// Only target <namespace>/<namespace>.go files
 		if !d.IsDir() && strings.TrimSuffix(d.Name(), ".go") == filepath.Base(filepath.Dir(p)) {
 			files = append(files, p)
 			fmt.Printf("Successfully bisynced %s\n", logging.Bold(p))
@@ -83,7 +81,7 @@ func BisyncGo(ctx context.Context, outputPath, lekkoPath, repoPath string) ([]st
 		return nil, errors.Wrap(err, "sync")
 	}
 	if err := WriteContentsToLocalRepo(ctx, repoContents, repoPath); err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, "write to repository")
 	}
 	generator, err := gen.NewGoGenerator(mf.Module.Mod.Path, outputPath, lekkoPath, repoContents)
 	if err != nil {
@@ -324,7 +322,6 @@ func (g *goSyncer) AstToNamespace(pf *ast.File, fds *descriptorpb.FileDescriptor
 
 // Translate a collection of Go files to a representation of repository contents.
 // Files -> repo instead of file -> namespace because FDS is shared repo-wide.
-// Takes file paths instead of contents for more helpful error reporting.
 func (g *goSyncer) Sync(filePaths ...string) (*featurev1beta1.RepositoryContents, error) {
 	ret := &featurev1beta1.RepositoryContents{FileDescriptorSet: protoutils.NewDefaultFileDescriptorSet()}
 	for _, filePath := range filePaths {
@@ -342,6 +339,23 @@ func (g *goSyncer) Sync(filePaths ...string) (*featurev1beta1.RepositoryContents
 	return ret, nil
 }
 
+func (g *goSyncer) SyncContents(contentMap map[string]string) (*featurev1beta1.RepositoryContents, error) {
+	ret := &featurev1beta1.RepositoryContents{FileDescriptorSet: protoutils.NewDefaultFileDescriptorSet()}
+	for namespace, contents := range contentMap {
+		astf, err := parser.ParseFile(g.fset, namespace, contents, parser.ParseComments|parser.AllErrors|parser.SkipObjectResolution)
+		if err != nil {
+			return nil, errors.Wrapf(err, "parse %s", namespace)
+		}
+		ns, err := g.AstToNamespace(astf, ret.FileDescriptorSet)
+		if err != nil {
+			return nil, errors.Wrapf(err, "translate %s", namespace)
+		}
+		ret.Namespaces = append(ret.Namespaces, ns)
+	}
+
+	return ret, nil
+}
+
 // TODO - is this only used for context keys, or other things?
 func (g *goSyncer) exprToValue(expr ast.Expr) string {
 	//fmt.Printf("%+v\n", expr)

pkg/sync/push.go

@@ -115,7 +115,7 @@ func Push(ctx context.Context, commitMessage string, force bool, dot *dotlekko.D
 	}))
 	switch nlProject.Language {
 	case native.LangTypeScript:
-		err = BisyncTS(lekkoPath, repoPath)
+		err = BisyncTS(ctx, lekkoPath, repoPath)
 		if err != nil {
 			return err
 		}

pkg/sync/repo.go

@@ -34,6 +34,10 @@ import (
 	"google.golang.org/protobuf/types/known/anypb"
 )
 
+// Writes contents to a local repository at the specified path.
+// Conflicting namespaces are essentially completely overwritten.
+// Namespaces in the local repository but not in the passed contents are untouched.
+// New namespaces are created.
 func WriteContentsToLocalRepo(ctx context.Context, contents *featurev1beta1.RepositoryContents, repoPath string) error {
 	// NOTE: For now, this function still needs a proper Lekko repository as a prereq,
 	// because it's uncertain if we'll ever need functionality to create a local repository

pkg/sync/ts.go

@@ -15,22 +15,84 @@
 package sync
 
 import (
-	"fmt"
+	"context"
+	"encoding/base64"
 	"os/exec"
+	"path/filepath"
 	"strings"
 
+	"github.com/lekkodev/cli/pkg/gen"
+	"github.com/lekkodev/cli/pkg/native"
+	protoutils "github.com/lekkodev/cli/pkg/proto"
 	"github.com/pkg/errors"
+	"google.golang.org/protobuf/proto"
+
+	featurev1beta1 "buf.build/gen/go/lekkodev/cli/protocolbuffers/go/lekko/feature/v1beta1"
 )
 
-func BisyncTS(lekkoPath, repoPath string) error {
-	tsSyncCmd := exec.Command("npx", "lekko-repo-sync", "--lekko-dir", lekkoPath, "--repo-path", repoPath)
+// TODO: consider consolidating bisync methods
+func BisyncTS(ctx context.Context, lekkoPath, repoPath string) error {
+	repoContents, err := SyncTS(lekkoPath)
+	if err != nil {
+		return errors.Wrap(err, "sync")
+	}
+	if err := WriteContentsToLocalRepo(ctx, repoContents, repoPath); err != nil {
+		return errors.Wrap(err, "write to repository")
+	}
+	// Generate only for namespaces that were synced
+	for _, namespace := range repoContents.Namespaces {
+		outFilename := filepath.Join(lekkoPath, namespace.Name+native.LangTypeScript.Ext())
+		if err := gen.GenFormattedTS(ctx, repoPath, namespace.Name, outFilename); err != nil {
+			return errors.Wrapf(err, "generate code for %s", namespace.Name)
+		}
+	}
+	return nil
+}
+
+func SyncTS(lekkoPath string) (*featurev1beta1.RepositoryContents, error) {
+	// TODO: If command not found, give helpful error message prompting user to update dependencies
+	tsSyncCmd := exec.Command("npx", "lekko-sync", "--lekko-dir", lekkoPath)
 	output, err := tsSyncCmd.CombinedOutput()
-	outputStr := strings.TrimSpace(string(output))
-	if len(outputStr) > 0 {
-		fmt.Println(outputStr)
+	if err != nil {
+		return nil, errors.Wrapf(err, "sync ts: %s", output)
 	}
+	return syncTSOutputToRepositoryContents(output)
+}
+
+func SyncTSFiles(lekkoFiles ...string) (*featurev1beta1.RepositoryContents, error) {
+	commafied := strings.Join(lekkoFiles, ",")
+	tsSyncCmd := exec.Command("npx", "lekko-sync", "--lekko-files", commafied)
+	output, err := tsSyncCmd.CombinedOutput()
 	if err != nil {
-		return errors.Wrap(err, "bisync ts")
+		return nil, errors.Wrapf(err, "sync ts: %s", output)
 	}
-	return nil
+	return syncTSOutputToRepositoryContents(output)
+}
+
+// Output is expected to be base64 encoded serialized RepositoryContents message
+func syncTSOutputToRepositoryContents(output []byte) (*featurev1beta1.RepositoryContents, error) {
+	// Because Protobuf is not self-describing, we have to jump through some hoops here for deserialization.
+	// The RepositoryContents message contains the FDS which we want to use as the resolver for unmarshalling
+	// the rest of the contents.
+	decoded, err := base64.StdEncoding.DecodeString(string(output))
+	if err != nil {
+		return nil, errors.Wrap(err, "decode sync ts output")
+	}
+	// First pass unmarshal to get the FDS while ignoring any unresolvable Anys
+	tempRepoContents := &featurev1beta1.RepositoryContents{}
+	err = proto.UnmarshalOptions{DiscardUnknown: true, Resolver: &protoutils.IgnoreAnyResolver{}}.Unmarshal(decoded, tempRepoContents)
+	if err != nil {
+		return nil, errors.Wrap(err, "initial unmarshal sync ts output")
+	}
+	typeRegistry, err := protoutils.FileDescriptorSetToTypeRegistry(tempRepoContents.FileDescriptorSet)
+	if err != nil {
+		return nil, errors.Wrap(err, "get type registry")
+	}
+	// Re-unmarshal using type registry this time - we can resolve Anys correctly
+	repoContents := &featurev1beta1.RepositoryContents{}
+	err = proto.UnmarshalOptions{Resolver: typeRegistry}.Unmarshal(decoded, repoContents)
+	if err != nil {
+		return nil, errors.Wrap(err, "final unmarshal sync ts output")
+	}
+	return repoContents, nil
 }