Merge pull request #93 from drmaas/map_lists_disable_rpcs

Allow maps of repeated items, add some compile options

.gitignore

@@ -1,2 +1,3 @@
 *.swp
 .cover
+.idea

README.md

@@ -14,8 +14,10 @@ go get -u github.com/NYTimes/openapi2proto/cmd/openapi2proto
 There are 4 CLI flags for using the tool:
 * `-spec` to point to the appropriate OpenAPI spec file
 * `-annotate` to include (google.api.http options) for [grpc-gateway](https://github.com/gengo/grpc-gateway) users. This is disabled by default.
-* `-out` to have the output written to a file rather than `Stdout. Defaults to `Stdout` if this is not specified`
+* `-out` to have the output written to a file rather than `Stdout`. Defaults to `Stdout` if this is not specified.
 * `-indent` to override the default indentation for Protobuf specs of 4 spaces.
+* `-skip-rpcs` to skip generation of rpcs. These are generated by default.
+* `-namespace-enums` to enable inserting the enum name as an enum prefix for each value. This is disabled by default.
 
 ## Protobuf Tags
 * To allow for more control over how your protobuf schema evolves, all parameters and property definitions will accept an optional extension parameter, `x-proto-tag`, that will overide the generated tag with the value supplied.

cmd/openapi2proto/main.go

@@ -22,9 +22,11 @@ func main() {
 
 func _main() error {
 	specPath := flag.String("spec", "../../spec.yaml", "location of the swagger spec file")
-	annotate := flag.Bool("annotate", false, "include (google.api.http) options for grpc-gateway")
+	annotate := flag.Bool("annotate", false, "include (google.api.http) options for grpc-gateway. Defaults to false if not set")
 	outfile := flag.String("out", "", "the file to output the result to. Defaults to stdout if not set")
 	indent := flag.Int("indent", 4, "number of spaces used for indentation")
+	skipRpcs := flag.Bool("skip-rpcs", false, "skip rpc code generation. Defaults to false if not set")
+	namespaceEnums := flag.Bool("namespace-enums", false, "prefix enum values with the enum name to prevent namespace conflicts. Defaults to false if not set")
 	flag.Parse()
 
 	var dst io.Writer = os.Stdout
@@ -42,6 +44,8 @@ func _main() error {
 	var compilerOptions []compiler.Option
 
 	compilerOptions = append(compilerOptions, compiler.WithAnnotation(*annotate))
+	compilerOptions = append(compilerOptions, compiler.WithSkipRpcs(*skipRpcs))
+	compilerOptions = append(compilerOptions, compiler.WithPrefixEnums(*namespaceEnums))
 
 	if *indent > 0 {
 		var indentStr bytes.Buffer

compiler/compiler.go

@@ -28,6 +28,7 @@ var knownImports = map[string]string{
 	"google.protobuf.NullValue":     "google/protobuf/struct.proto",
 	"google.protobuf.MethodOptions": "google/protobuf/descriptor.proto",
 	"google.protobuf.Timestamp":     "google/protobuf/timestamp.proto",
+	"google.protobuf.ListValue":     "google/protobuf/struct.proto",
 }
 
 var knownDefinitions = map[string]protobuf.Type{}
@@ -48,15 +49,23 @@ func newCompileCtx(spec *openapi.Spec, options ...Option) *compileCtx {
 	p.AddType(svc)
 
 	var annotate bool
+	var skipRpcs bool
+	var namespaceEnums bool
 	for _, o := range options {
 		switch o.Name() {
 		case optkeyAnnotation:
 			annotate = o.Value().(bool)
+		case optkeySkipRpcs:
+			skipRpcs = o.Value().(bool)
+		case optkeyPrefixEnums:
+			namespaceEnums = o.Value().(bool)
 		}
 	}
 
 	c := &compileCtx{
 		annotate:            annotate,
+		skipRpcs:            skipRpcs,
+		namespaceEnums:      namespaceEnums,
 		definitions:         map[string]protobuf.Type{},
 		externalDefinitions: map[string]map[string]protobuf.Type{},
 		imports:             map[string]struct{}{},
@@ -67,6 +76,7 @@ func newCompileCtx(spec *openapi.Spec, options ...Option) *compileCtx {
 		service:             svc,
 		types:               map[protobuf.Container]map[protobuf.Type]struct{}{},
 		unfulfilledRefs:     map[string]struct{}{},
+		messageNames:        map[string]bool{},
 	}
 	return c
 }
@@ -109,9 +119,11 @@ func Compile(spec *openapi.Spec, options ...Option) (*protobuf.Package, error) {
 	}
 
 	// compile the paths
-	c.phase = phaseCompilePaths
-	if err := c.compilePaths(spec.Paths); err != nil {
-		return nil, errors.Wrap(err, `failed to compile paths`)
+	if !c.skipRpcs {
+		c.phase = phaseCompilePaths
+		if err := c.compilePaths(spec.Paths); err != nil {
+			return nil, errors.Wrap(err, `failed to compile paths`)
+		}
 	}
 
 	return c.pkg, nil
@@ -435,7 +447,7 @@ func (c *compileCtx) getTypeFromReference(ref string) (protobuf.Type, error) {
 
 func (c *compileCtx) compileEnum(name string, elements []string) (*protobuf.Enum, error) {
 	var prefix bool
-	if c.parent() != c.pkg {
+	if c.parent() != c.pkg || c.namespaceEnums {
 		prefix = true
 	}
 
@@ -449,7 +461,6 @@ func (c *compileCtx) compileEnum(name string, elements []string) (*protobuf.Enum
 
 		e.AddElement(allCaps(ename))
 	}
-
 	return e, nil
 }
 
@@ -477,7 +488,7 @@ func (c *compileCtx) compileSchemaMultiType(name string, s *openapi.Schema) (pro
 	return c.getBoxedType(c.applyBuiltinFormat(v, s.Format)), nil
 }
 
-func (c *compileCtx) compileMap(name string, s *openapi.Schema) (protobuf.Type, error) {
+func (c *compileCtx) compileMap(name string, rawName string, s *openapi.Schema) (protobuf.Type, error) {
 	var typ protobuf.Type
 
 	switch {
@@ -489,9 +500,35 @@ func (c *compileCtx) compileMap(name string, s *openapi.Schema) (protobuf.Type,
 		}
 	case !s.Type.Empty():
 		var err error
-		typ, err = c.getType(s.Type.First())
-		if err != nil {
-			return nil, errors.Wrapf(err, `failed to get type %s`, s.Type)
+		if s.Type.First() == "array" && s.Items != nil {
+			if s.Items.Ref != "" {
+				// reference schema for array items
+				baseFieldName := camelCase(strings.TrimPrefix(s.Items.Ref, "#/definitions"))
+				typ = c.createListWrapper(name, rawName, baseFieldName, s)
+				// finally, make sure that this type is registered, if need be.
+				c.addTypeToParent(typ, c.grandParent())
+			} else if !s.Items.Type.Empty() && (s.Items.Properties == nil || len(s.Items.Properties) == 0) {
+				// inline object for array of untyped items
+				typ = protobuf.ListValueType
+				c.addImportForType(typ.Name())
+			} else if !s.Items.Type.Empty() && len(s.Items.Properties) > 0 {
+				// inline object for array of typed items
+				baseFieldName := camelCase(name)
+				typ = c.createListWrapper(name, rawName, baseFieldName, s)
+				// finally, make sure that this type is registered, if need be.
+				c.addType(typ)
+				subtyp, err := c.compileSchema(name, s.Items)
+				if err == nil {
+					c.addType(subtyp)
+				}
+			} else {
+				return nil, errors.Errorf(`An array for map types must specify a reference or an object`)
+			}
+		} else {
+			typ, err = c.getType(s.Type.First())
+			if err != nil {
+				return nil, errors.Wrapf(err, `failed to get type %s`, s.Type)
+			}
 		}
 	default:
 		var err error
@@ -501,7 +538,6 @@ func (c *compileCtx) compileMap(name string, s *openapi.Schema) (protobuf.Type,
 		}
 	}
 
-	// Note: Map of arrays is not currently supported.
 	return protobuf.NewMap(protobuf.StringType, typ), nil
 
 }
@@ -534,7 +570,6 @@ func (c *compileCtx) compileSchema(name string, s *openapi.Schema) (protobuf.Typ
 		}
 		return m, nil
 	}
-
 	rawName := name
 	name = camelCase(name)
 	// could be a builtin... try as-is once, then the camel cased
@@ -555,7 +590,7 @@ func (c *compileCtx) compileSchema(name string, s *openapi.Schema) (protobuf.Typ
 	switch {
 	case s.Type.Empty() || s.Type.Contains("object"):
 		if ap := s.AdditionalProperties; ap != nil && !ap.IsNil() {
-			return c.compileMap(name, ap)
+			return c.compileMap(name, strings.TrimSuffix(rawName, "Message"), ap)
 		}
 
 		m := protobuf.NewMessage(name)
@@ -833,6 +868,15 @@ func (c *compileCtx) parent() protobuf.Container {
 	return c.parents[l-1]
 }
 
+func (c *compileCtx) grandParent() protobuf.Container {
+	switch len(c.parents) {
+	case 0:
+		return c.pkg
+	default:
+		return c.parents[0]
+	}
+}
+
 // adds new type. dedupes, in case of multiple addition
 func (c *compileCtx) addType(t protobuf.Type) {
 	c.addTypeToParent(t, c.parent())
@@ -854,6 +898,22 @@ func (c *compileCtx) addTypeToParent(t protobuf.Type, p protobuf.Container) {
 		}
 	}
 
+	// hack alert - check for duplicates
+	// I couldn't figure out how to stop map list value wrappers from being specified more than once.
+	// This is generalized here based on the type hierarchy to prevent duplicates of all messages.
+	parentNames := func(vs []protobuf.Container) []string {
+		vsm := make([]string, len(vs))
+		for i, v := range vs {
+			vsm[i] = v.Name()
+		}
+		return vsm
+	}(c.parents)
+	key := strings.Trim(strings.Join(parentNames, "#"), "[]") + "#" + t.Name()
+	if _, ok := c.messageNames[key]; ok {
+		return
+	}
+	c.messageNames[key] = true
+
 	m, ok := c.types[p]
 	if !ok {
 		m = map[protobuf.Type]struct{}{}
@@ -903,9 +963,26 @@ func (c *compileCtx) compilePaths(paths map[string]*openapi.Path) error {
 	return nil
 }
 
+func (c *compileCtx) createListWrapper(name string, rawName string, baseFieldName string, s *openapi.Schema) protobuf.Type {
+	// we need to construct a new statically typed wrapper message that contains a repeated list of items
+	// referenced by the spec
+	mapValueName := strings.TrimSuffix(name, "Message") + "List"
+	m := protobuf.NewMessage(mapValueName)
+	f := protobuf.NewField(protobuf.NewMessage(baseFieldName), rawName, 1)
+	f.SetRepeated(true)
+	if v := s.Description; len(v) > 0 {
+		f.SetComment(v)
+	}
+	m.AddField(f)
+	m.SetComment("automatically generated wrapper for a list of " + baseFieldName + " items")
+	return m
+}
+
 func mergeParameters(p1, p2 openapi.Parameters) openapi.Parameters {
 	var out openapi.Parameters
 	out = append(out, p1...)
 	out = append(out, p2...)
 	return out
 }
+
+

compiler/interface.go

@@ -18,6 +18,8 @@ type Option = option.Option
 
 type compileCtx struct {
 	annotate            bool
+	skipRpcs            bool
+	namespaceEnums      bool
 	definitions         map[string]protobuf.Type
 	externalDefinitions map[string]map[string]protobuf.Type
 	imports             map[string]struct{}
@@ -29,4 +31,5 @@ type compileCtx struct {
 	service             *protobuf.Service
 	types               map[protobuf.Container]map[protobuf.Type]struct{}
 	unfulfilledRefs     map[string]struct{}
+	messageNames        map[string]bool
 }

compiler/options.go

@@ -3,11 +3,23 @@ package compiler
 import "github.com/NYTimes/openapi2proto/internal/option"
 
 const (
-	optkeyAnnotation = "annotation"
+	optkeyAnnotation  = "annotation"
+	optkeySkipRpcs    = "skip-rpcs"
+	optkeyPrefixEnums = "namespace-enums"
 )
 
 // WithAnnotation creates a new Option to specify if we should add
 // google.api.http annotation to the compiled Protocol Buffers structure
 func WithAnnotation(b bool) Option {
 	return option.New(optkeyAnnotation, b)
 }
+
+// WithSkipRpcs creates a new Option to specify if we should
+// generate services and rpcs in addition to messages
+func WithSkipRpcs(b bool) Option {
+	return option.New(optkeySkipRpcs, b)
+}
+
+func WithPrefixEnums(b bool) Option {
+	return option.New(optkeyPrefixEnums, b)
+}

fixtures/refs.json

@@ -5,23 +5,58 @@
   },
   "definitions": {
     "TestModel": {
-        "type": "object",
-        "properties": {
-            "test_map_object": {
-                "type": "object",
-                "additionalProperties": {
-                    "$ref": "#/definitions/TestModel"
-                },
-                "x-proto-tag": 11
-            },
-            "test_map_scalar": {
-                "type": "object",
-                "additionalProperties": {
-                    "type": "string"
-                },
-                "x-proto-tag": 12
+      "type": "object",
+      "properties": {
+        "test_map_object": {
+          "type": "object",
+          "additionalProperties": {
+            "$ref": "#/definitions/TestModel"
+          },
+          "x-proto-tag": 11
+        },
+        "test_map_scalar": {
+          "type": "object",
+          "additionalProperties": {
+            "type": "string"
+          },
+          "x-proto-tag": 12
+        },
+        "test_map_array": {
+          "type": "object",
+          "additionalProperties": {
+            "type": "array",
+            "items": {
+              "$ref": "#/definitions/TestModel"
             }
+          },
+          "x-proto-tag": 13
+        },
+        "test_map_array_untyped_fields": {
+          "type": "object",
+          "additionalProperties": {
+            "type": "array",
+            "items": {
+              "type": "object",
+              "properties": {
+                "id": {
+                  "type": "integer"
+                }
+              }
+            }
+          },
+          "x-proto-tag": 15
+        },
+        "test_map_array_untyped": {
+          "type": "object",
+          "additionalProperties": {
+            "type": "array",
+            "items": {
+              "type": "object"
+            }
+          },
+          "x-proto-tag": 14
         }
+      }
     }
   }
 }

fixtures/refs.proto

@@ -2,7 +2,26 @@ syntax = "proto3";
 
 package refs;
 
+import "google/protobuf/struct.proto";
+
+// automatically generated wrapper for a list of TestModel items
+message TestMapArrayList {
+    repeated TestModel test_map_array = 1;
+}
+
 message TestModel {
+    // automatically generated wrapper for a list of TestMapArrayUntypedFieldsMessage items
+    message TestMapArrayUntypedFieldsList {
+        repeated TestMapArrayUntypedFieldsMessage test_map_array_untyped_fields = 1;
+    }
+
+    message TestMapArrayUntypedFieldsMessage {
+        int32 id = 1;
+    }
+
     map<string, TestModel> test_map_object = 11;
     map<string, string> test_map_scalar = 12;
+    map<string, TestMapArrayList> test_map_array = 13;
+    map<string, google.protobuf.ListValue> test_map_array_untyped = 14;
+    map<string, TestMapArrayUntypedFieldsList> test_map_array_untyped_fields = 15;
 }