Feature: add pkg/process (#21)

* Initial pkg/starlarkprocess

* Replace skycfg with deterministic fork

* Try fix flaky tests

* More test fix for flakiness

* revert use fork

* Revert use of fork; try fixing up prototext output directly

* Yes more flaky reduce efforts

* Add

Makefile

@@ -28,6 +28,7 @@ serve: build
 routeguide_proto_descriptor:
 	bazel build //example/routeguide:routeguide_proto_descriptor
 	cp -f bazel-bin/example/routeguide/routeguide_proto_descriptor.pb pkg/starlarkgrpc/
+	cp -f bazel-bin/example/routeguide/routeguide_proto_descriptor.pb pkg/protodescriptorset/
 
 .PHONY: plugin_proto_descriptor
 plugin_proto_descriptor:

cmd/grpcstar/BUILD.bazel

@@ -67,7 +67,7 @@ go_binary(
 
 go_test(
     name = "grpcstar_test",
-    srcs = ["integration_test.go"],
+    srcs = ["grpcstar_test.go"],
     data = glob(["testdata/**"]) + ["//example/routeguide:routeguide_proto_descriptor"],
     embed = [":grpcstar_lib"],
     deps = [

cmd/grpcstar/integration_test.go → cmd/grpcstar/grpcstar_test.go

@@ -4,6 +4,7 @@ import (
 	"flag"
 	"os"
 	"path/filepath"
+	"regexp"
 	"strings"
 	"testing"
 	"time"
@@ -93,6 +94,7 @@ func TestGoldens(t *testing.T) {
 			if err != nil {
 				t.Fatal("reading err file:", err)
 			}
+			gotErr = derandPrototext(t, gotErr)
 
 			if *update {
 				if workspaceDir == "" {
@@ -126,3 +128,11 @@ func TestGoldens(t *testing.T) {
 		})
 	}
 }
+
+func derandPrototext(t *testing.T, data []byte) []byte {
+	in := string(data)
+	re := regexp.MustCompile(`\s{2}([a-z]+):`)
+	out := re.ReplaceAllString(in, " $1:")
+	out = re.ReplaceAllString(out, " $1:")
+	return []byte(out)
+}

cmd/grpcstar/testdata/headers.grpc.star.err

@@ -1,4 +1,4 @@
-[testdata/headers.grpc.star:22:10] server: GetFeature request message: <example.routeguide.Point latitude:2  longitude:1>
+[testdata/headers.grpc.star:22:10] server: GetFeature request message: <example.routeguide.Point latitude:2 longitude:1>
 [testdata/headers.grpc.star:23:10] server: GetFeature request headers: [":authority", "content-type", "user-agent", "x-unary-request"]
 [testdata/headers.grpc.star:24:10] server: GetFeature request header content-type: application/grpc
 [testdata/headers.grpc.star:25:10] server: GetFeature request header user-agent: grpc-go/1.35.0

cmd/grpcstar/testdata/headers.grpc.star.out

@@ -1,2 +1,4 @@
-{"name":"point (1,2)"}
+{
+ "name": "point (1,2)"
+}
 {}

cmd/grpcstar/testdata/process.grpc.star

@@ -0,0 +1,11 @@
+def main(ctx):
+    print(process)
+
+    print("run:", process.run)
+    result = process.run(
+        command = "pwd",
+    )
+    print("stdout (runfiles dir):", str(result.stdout).partition("grpcstar_test.runfiles")[2])
+    print("stderr:", result.stderr)
+    print("error:", result.error)
+    print("exit_code:", result.exit_code)

cmd/grpcstar/testdata/process.grpc.star.err

@@ -0,0 +1,7 @@
+[testdata/process.grpc.star:2:10] <module "process">
+[testdata/process.grpc.star:4:10] run: <built-in function process.run>
+[testdata/process.grpc.star:8:10] stdout (runfiles dir): /build_stack_grpc_starlark/cmd/grpcstar
+
+[testdata/process.grpc.star:9:10] stderr: 
+[testdata/process.grpc.star:10:10] error: 
+[testdata/process.grpc.star:11:10] exit_code: 0

cmd/grpcstar/testdata/routeguide.grpc.star.err

@@ -1,6 +1,6 @@
 [testdata/routeguide.grpc.star:92:10] GetFeature: <example.routeguide.Feature name:"point (1,2)">
 [testdata/routeguide.grpc.star:104:14] ListFeatures: <example.routeguide.Feature name:"lo (1,2)">
 [testdata/routeguide.grpc.star:104:14] ListFeatures: <example.routeguide.Feature name:"hi (1,4)">
-[testdata/routeguide.grpc.star:113:10] RecordRoute: <example.routeguide.RouteSummary point_count:2  distance:2  elapsed_time:10>
+[testdata/routeguide.grpc.star:113:10] RecordRoute: <example.routeguide.RouteSummary point_count:2 distance:2 elapsed_time:10>
 [testdata/routeguide.grpc.star:124:14] RouteChat: <example.routeguide.RouteNote message:"A">
 [testdata/routeguide.grpc.star:124:14] RouteChat: <example.routeguide.RouteNote message:"B">

cmd/grpcstar/testdata/routeguide.grpc.star.out

@@ -1,6 +1,20 @@
-{"name":"point (1,2)"}
-{"name":"lo (1,2)"}
-{"name":"hi (1,4)"}
-{"point_count":2, "distance":2, "elapsed_time":10}
-{"message":"A"}
-{"message":"B"}
+{
+ "name": "point (1,2)"
+}
+{
+ "name": "lo (1,2)"
+}
+{
+ "name": "hi (1,4)"
+}
+{
+ "point_count": 2,
+ "distance": 2,
+ "elapsed_time": 10
+}
+{
+ "message": "A"
+}
+{
+ "message": "B"
+}

cmd/grpcstar/testdata/thread.grpc.star

@@ -10,9 +10,9 @@ def main(ctx):
     > thread.sleep pauses the current thread for the given duration
     """)
     print("thread.sleep:", thread.sleep)
-    thread.sleep(duration = 200 * time.millisecond)
+    thread.sleep(duration = 100 * time.millisecond)
     print("sleep before:", now)
-    print("sleep after:", time.now())
+    # print("sleep after:", time.now())
 
     print("""
     > thread.defer runs a function in a separate thread after a given delay
@@ -21,7 +21,7 @@ def main(ctx):
         # FIXME(pcj): figure out how to make this non-flaky
         # fn = lambda: print("defer at %s in thread %s:" % (time.now(), thread.name())),
         fn = lambda: print("defer callback in thread %s:" % thread.name()),
-        delay = 150 * time.millisecond,
+        delay = 10 * time.millisecond,
         count = 3,
     )
 

cmd/grpcstar/testdata/thread.grpc.star.err

@@ -7,13 +7,12 @@
 
 [testdata/thread.grpc.star:12:10] thread.sleep: <built-in function thread.sleep>
 [testdata/thread.grpc.star:14:10] sleep before: 2019-01-01 00:00:00 +0000 UTC
-[testdata/thread.grpc.star:15:10] sleep after: 2019-01-01 00:00:00.2 +0000 UTC
 [testdata/thread.grpc.star:17:10] 
     > thread.defer runs a function in a separate thread after a given delay
 
-[testdata/thread.grpc.star:23:27] defer callback in thread -thread.defer(150000000):
-[testdata/thread.grpc.star:23:27] defer callback in thread -thread.defer(150000000):
-[testdata/thread.grpc.star:23:27] defer callback in thread -thread.defer(150000000):
+[testdata/thread.grpc.star:23:27] defer callback in thread -thread.defer(10000000):
+[testdata/thread.grpc.star:23:27] defer callback in thread -thread.defer(10000000):
+[testdata/thread.grpc.star:23:27] defer callback in thread -thread.defer(10000000):
 [testdata/thread.grpc.star:31:10] 
     > thread.cancel stops the current thread.  Cancelling the main thread exits the program.
 

cmd/grpcstar/testdata/time.grpc.star

@@ -1,8 +1,10 @@
 def main(ctx):
     print("=== Example Time Usage ===")
     now = time.now()
+    then = now + 5 * time.second
 
-    print("current time:", now)
-    print("time add:", now + 5 * time.second)
-    print("time hours:", now.hour)
+    print("time then:", then)
+    print("time hours:", then.hour)
+    print("time minute:", then.minute)
+    print("time second:", then.second)
     print("additional details: https://github.com/google/starlark-go/blob/a134d8f9ddca7469c736775b67544671f0a135ad/starlark/testdata/time.star")

cmd/grpcstar/testdata/time.grpc.star.err

@@ -1,5 +1,6 @@
 [testdata/time.grpc.star:2:10] === Example Time Usage ===
-[testdata/time.grpc.star:5:10] current time: 2019-01-01 00:00:00.7 +0000 UTC
-[testdata/time.grpc.star:6:10] time add: 2019-01-01 00:00:05.7 +0000 UTC
+[testdata/time.grpc.star:6:10] time then: 2019-01-01 00:00:05.6 +0000 UTC
 [testdata/time.grpc.star:7:10] time hours: 0
-[testdata/time.grpc.star:8:10] additional details: https://github.com/google/starlark-go/blob/a134d8f9ddca7469c736775b67544671f0a135ad/starlark/testdata/time.star
+[testdata/time.grpc.star:8:10] time minute: 0
+[testdata/time.grpc.star:9:10] time second: 5
+[testdata/time.grpc.star:10:10] additional details: https://github.com/google/starlark-go/blob/a134d8f9ddca7469c736775b67544671f0a135ad/starlark/testdata/time.star

pkg/program/BUILD.bazel

@@ -15,6 +15,7 @@ go_library(
         "//pkg/starlarkgrpc",
         "//pkg/starlarknet",
         "//pkg/starlarkos",
+        "//pkg/starlarkprocess",
         "//pkg/starlarkthread",
         "@com_github_stripe_skycfg//:skycfg",
         "@com_github_stripe_skycfg//go/protomodule",

pkg/program/config.go

@@ -88,7 +88,7 @@ func (cfg *Config) ParseArgs(args []string) error {
 	case OutputStableJson:
 		marshaler := protojson.MarshalOptions{
 			EmitUnpopulated: false,
-			Indent:          "",
+			Indent:          " ",
 			UseProtoNames:   true,
 		}
 		cfg.OutputType = OutputStableJson
@@ -130,12 +130,14 @@ func (cfg *Config) ParseArgs(args []string) error {
 			return yaml.Marshal(yamlMap)
 		}
 	default:
-		return Usage(fmt.Sprintf("invalid flag -o: must be one of (%v|%v|%v|%v)",
-			OutputJson,
-			OutputStableJson,
-			OutputProto,
-			OutputText,
-			OutputYaml,
+		return Usage(fmt.Sprintf("invalid flag -o: must be one of (%v)",
+			[]OutputType{
+				OutputJson,
+				OutputStableJson,
+				OutputProto,
+				OutputText,
+				OutputYaml,
+			},
 		))
 	}
 

pkg/program/program.go

@@ -18,6 +18,7 @@ import (
 	"github.com/stackb/grpc-starlark/pkg/starlarkgrpc"
 	"github.com/stackb/grpc-starlark/pkg/starlarknet"
 	"github.com/stackb/grpc-starlark/pkg/starlarkos"
+	"github.com/stackb/grpc-starlark/pkg/starlarkprocess"
 	"github.com/stackb/grpc-starlark/pkg/starlarkthread"
 )
 
@@ -26,14 +27,18 @@ type Program struct {
 	skyConfig *skycfg.Config
 }
 
-func NewProgram(cfg *Config) (*Program, error) {
+func NewProgram(cfg *Config, loadOptions ...skycfg.LoadOption) (*Program, error) {
 	if cfg.File == "" {
 		return nil, fmt.Errorf("entrypoint file is required")
 	}
-	skyConfig, err := skycfg.Load(context.Background(), cfg.File,
+	loadOptions = append(loadOptions,
 		skycfg.WithProtoRegistry(skycfg.NewUnstableProtobufRegistryV2(cfg.ProtoTypes)),
+	)
+	loadOptions = append(loadOptions,
 		skycfg.WithGlobals(newPredeclared(cfg.ProtoFiles, cfg.ProtoTypes)),
 	)
+
+	skyConfig, err := skycfg.Load(context.Background(), cfg.File, loadOptions...)
 	if err != nil {
 		return nil, err
 	}
@@ -52,7 +57,7 @@ func (p *Program) Run(options ...skycfg.ExecOption) error {
 		}
 		return err
 	}
-	if err := p.Format(msgs); err != nil {
+	if err := p.Format(msgs...); err != nil {
 		return err
 	}
 	return nil
@@ -69,7 +74,7 @@ func (p *Program) Exec() ([]protoreflect.ProtoMessage, error) {
 	return msgs, nil
 }
 
-func (p *Program) Format(msgs []protoreflect.ProtoMessage) error {
+func (p *Program) Format(msgs ...protoreflect.ProtoMessage) error {
 	var sep string
 	if p.cfg.OutputType == OutputYaml {
 		sep = "---\n"
@@ -80,7 +85,9 @@ func (p *Program) Format(msgs []protoreflect.ProtoMessage) error {
 			return err
 		}
 		if p.cfg.OutputType == OutputProto {
-			fmt.Print(data)
+			if _, err := os.Stdout.Write(data); err != nil {
+				return err
+			}
 		} else {
 			fmt.Printf("%s%s\n", sep, string(data))
 		}
@@ -110,14 +117,15 @@ func newPredeclared(files *protoregistry.Files, types *protoregistry.Types) star
 	protoModule.Members["decode"] = protoDecode(types)
 
 	return starlark.StringDict{
-		"os":     starlarkos.Module,
-		"net":    starlarknet.Module,
-		"thread": starlarkthread.Module,
-		"time":   libtime.Module,
-		"crypto": starlarkcrypto.Module,
-		"grpc":   starlarkgrpc.NewModule(files),
-		"proto":  protoModule,
-		"struct": starlark.NewBuiltin("struct", starlarkstruct.Make),
-		"module": starlark.NewBuiltin("module", starlarkstruct.MakeModule),
+		"os":      starlarkos.Module,
+		"net":     starlarknet.Module,
+		"thread":  starlarkthread.Module,
+		"time":    libtime.Module,
+		"crypto":  starlarkcrypto.Module,
+		"grpc":    starlarkgrpc.NewModule(files),
+		"proto":   protoModule,
+		"process": starlarkprocess.NewModule(),
+		"struct":  starlark.NewBuiltin("struct", starlarkstruct.Make),
+		"module":  starlark.NewBuiltin("module", starlarkstruct.MakeModule),
 	}
 }

pkg/program/proto.go

@@ -22,7 +22,7 @@ func protoDecode(registry *protoregistry.Types) starlark.Callable {
 		kwargs []starlark.Tuple,
 	) (starlark.Value, error) {
 		var msgType starlark.Value
-		var value starlark.String
+		var value starlark.Bytes
 		if err := starlark.UnpackPositionalArgs(fn.Name(), args, kwargs, 2, &msgType, &value); err != nil {
 			return nil, err
 		}
@@ -66,10 +66,11 @@ func protoEncode(registry *protoregistry.Types) starlark.Callable {
 			}
 		}
 
-		jsonData, err := marshal.Marshal(msg)
+		data, err := marshal.Marshal(msg)
 		if err != nil {
 			return nil, err
 		}
-		return starlark.String(jsonData), nil
+
+		return starlark.Bytes(data), nil
 	})
 }

pkg/protodescriptorset/BUILD.bazel

@@ -1,4 +1,4 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "protodescriptorset",
@@ -14,3 +14,16 @@ go_library(
         "@org_golang_google_protobuf//types/dynamicpb",
     ],
 )
+
+go_test(
+    name = "protodescriptorset_test",
+    srcs = ["protodescriptorset_test.go"],
+    embed = [":protodescriptorset"],
+    embedsrcs = ["routeguide_proto_descriptor.pb"],
+    deps = [
+        "@com_github_google_go_cmp//cmp",
+        "@org_golang_google_protobuf//reflect/protodesc",
+        "@org_golang_google_protobuf//reflect/protoreflect",
+        "@org_golang_google_protobuf//reflect/protoregistry",
+    ],
+)

pkg/protodescriptorset/protodescriptorset.go

@@ -49,6 +49,18 @@ func Parse(data []byte) (*descriptorpb.FileDescriptorSet, error) {
 	return &dpb, nil
 }
 
+func ParseFiles(data []byte) (*protoregistry.Files, error) {
+	descriptor, err := Parse(data)
+	if err != nil {
+		return nil, err
+	}
+	files, err := protodesc.NewFiles(descriptor)
+	if err != nil {
+		return nil, err
+	}
+	return files, nil
+}
+
 func FileTypes(files *protoregistry.Files) *protoregistry.Types {
 	var types protoregistry.Types
 	files.RangeFiles(func(fd protoreflect.FileDescriptor) bool {
@@ -69,3 +81,16 @@ func FileTypes(files *protoregistry.Files) *protoregistry.Types {
 	})
 	return &types
 }
+
+func MergeFilesIgnoreConflicts(all ...*protoregistry.Files) *protoregistry.Files {
+	merged := &protoregistry.Files{}
+	for _, files := range all {
+		files.RangeFiles(func(fd protoreflect.FileDescriptor) bool {
+			// RegisterFile only return err due to file or name conflicts.  This
+			// function is about ignoring conflicts, so we can ignore the error.
+			merged.RegisterFile(fd)
+			return true
+		})
+	}
+	return merged
+}