From cbdf4fd57b4e417147eeb23a5518319b8d9abdcf Mon Sep 17 00:00:00 2001 From: DanMaas Date: Tue, 18 Dec 2018 20:28:37 -0600 Subject: [PATCH] allow specification of a map of repeated items, add options to disable generation of rpcs and enable enum prefixes --- .gitignore | 1 + README.md | 4 +- cmd/openapi2proto/main.go | 6 ++- compiler/compiler.go | 101 +++++++++++++++++++++++++++++++++----- compiler/interface.go | 3 ++ compiler/options.go | 14 +++++- fixtures/refs.json | 65 ++++++++++++++++++------ fixtures/refs.proto | 19 +++++++ fixtures/refs.yaml | 24 +++++++++ protobuf/interface.go | 5 ++ 10 files changed, 212 insertions(+), 30 deletions(-) diff --git a/.gitignore b/.gitignore index 12afa21..b0aadec 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *.swp .cover +.idea diff --git a/README.md b/README.md index c1ee9cc..2b06d6b 100644 --- a/README.md +++ b/README.md @@ -14,8 +14,10 @@ go get -u github.com/NYTimes/openapi2proto/cmd/openapi2proto There are 4 CLI flags for using the tool: * `-spec` to point to the appropriate OpenAPI spec file * `-annotate` to include (google.api.http options) for [grpc-gateway](https://github.com/gengo/grpc-gateway) users. This is disabled by default. -* `-out` to have the output written to a file rather than `Stdout. Defaults to `Stdout` if this is not specified` +* `-out` to have the output written to a file rather than `Stdout`. Defaults to `Stdout` if this is not specified. * `-indent` to override the default indentation for Protobuf specs of 4 spaces. +* `-skip-rpcs` to skip generation of rpcs. These are generated by default. +* `-namespace-enums` to enable inserting the enum name as an enum prefix for each value. This is disabled by default. ## Protobuf Tags * To allow for more control over how your protobuf schema evolves, all parameters and property definitions will accept an optional extension parameter, `x-proto-tag`, that will overide the generated tag with the value supplied. diff --git a/cmd/openapi2proto/main.go b/cmd/openapi2proto/main.go index f314c32..0743e8b 100644 --- a/cmd/openapi2proto/main.go +++ b/cmd/openapi2proto/main.go @@ -22,9 +22,11 @@ func main() { func _main() error { specPath := flag.String("spec", "../../spec.yaml", "location of the swagger spec file") - annotate := flag.Bool("annotate", false, "include (google.api.http) options for grpc-gateway") + annotate := flag.Bool("annotate", false, "include (google.api.http) options for grpc-gateway. Defaults to false if not set") outfile := flag.String("out", "", "the file to output the result to. Defaults to stdout if not set") indent := flag.Int("indent", 4, "number of spaces used for indentation") + skipRpcs := flag.Bool("skip-rpcs", false, "skip rpc code generation. Defaults to false if not set") + namespaceEnums := flag.Bool("namespace-enums", false, "prefix enum values with the enum name to prevent namespace conflicts. Defaults to false if not set") flag.Parse() var dst io.Writer = os.Stdout @@ -42,6 +44,8 @@ func _main() error { var compilerOptions []compiler.Option compilerOptions = append(compilerOptions, compiler.WithAnnotation(*annotate)) + compilerOptions = append(compilerOptions, compiler.WithSkipRpcs(*skipRpcs)) + compilerOptions = append(compilerOptions, compiler.WithPrefixEnums(*namespaceEnums)) if *indent > 0 { var indentStr bytes.Buffer diff --git a/compiler/compiler.go b/compiler/compiler.go index 77a9f15..827e82e 100644 --- a/compiler/compiler.go +++ b/compiler/compiler.go @@ -28,6 +28,7 @@ var knownImports = map[string]string{ "google.protobuf.NullValue": "google/protobuf/struct.proto", "google.protobuf.MethodOptions": "google/protobuf/descriptor.proto", "google.protobuf.Timestamp": "google/protobuf/timestamp.proto", + "google.protobuf.ListValue": "google/protobuf/struct.proto", } var knownDefinitions = map[string]protobuf.Type{} @@ -48,15 +49,23 @@ func newCompileCtx(spec *openapi.Spec, options ...Option) *compileCtx { p.AddType(svc) var annotate bool + var skipRpcs bool + var namespaceEnums bool for _, o := range options { switch o.Name() { case optkeyAnnotation: annotate = o.Value().(bool) + case optkeySkipRpcs: + skipRpcs = o.Value().(bool) + case optkeyPrefixEnums: + namespaceEnums = o.Value().(bool) } } c := &compileCtx{ annotate: annotate, + skipRpcs: skipRpcs, + namespaceEnums: namespaceEnums, definitions: map[string]protobuf.Type{}, externalDefinitions: map[string]map[string]protobuf.Type{}, imports: map[string]struct{}{}, @@ -67,6 +76,7 @@ func newCompileCtx(spec *openapi.Spec, options ...Option) *compileCtx { service: svc, types: map[protobuf.Container]map[protobuf.Type]struct{}{}, unfulfilledRefs: map[string]struct{}{}, + messageNames: map[string]bool{}, } return c } @@ -109,9 +119,11 @@ func Compile(spec *openapi.Spec, options ...Option) (*protobuf.Package, error) { } // compile the paths - c.phase = phaseCompilePaths - if err := c.compilePaths(spec.Paths); err != nil { - return nil, errors.Wrap(err, `failed to compile paths`) + if !c.skipRpcs { + c.phase = phaseCompilePaths + if err := c.compilePaths(spec.Paths); err != nil { + return nil, errors.Wrap(err, `failed to compile paths`) + } } return c.pkg, nil @@ -435,7 +447,7 @@ func (c *compileCtx) getTypeFromReference(ref string) (protobuf.Type, error) { func (c *compileCtx) compileEnum(name string, elements []string) (*protobuf.Enum, error) { var prefix bool - if c.parent() != c.pkg { + if c.parent() != c.pkg || c.namespaceEnums { prefix = true } @@ -449,7 +461,6 @@ func (c *compileCtx) compileEnum(name string, elements []string) (*protobuf.Enum e.AddElement(allCaps(ename)) } - return e, nil } @@ -477,7 +488,7 @@ func (c *compileCtx) compileSchemaMultiType(name string, s *openapi.Schema) (pro return c.getBoxedType(c.applyBuiltinFormat(v, s.Format)), nil } -func (c *compileCtx) compileMap(name string, s *openapi.Schema) (protobuf.Type, error) { +func (c *compileCtx) compileMap(name string, rawName string, s *openapi.Schema) (protobuf.Type, error) { var typ protobuf.Type switch { @@ -489,9 +500,35 @@ func (c *compileCtx) compileMap(name string, s *openapi.Schema) (protobuf.Type, } case !s.Type.Empty(): var err error - typ, err = c.getType(s.Type.First()) - if err != nil { - return nil, errors.Wrapf(err, `failed to get type %s`, s.Type) + if s.Type.First() == "array" && s.Items != nil { + if s.Items.Ref != "" { + // reference schema for array items + baseFieldName := camelCase(strings.TrimPrefix(s.Items.Ref, "#/definitions")) + typ = c.createListWrapper(name, rawName, baseFieldName, s) + // finally, make sure that this type is registered, if need be. + c.addTypeToParent(typ, c.grandParent()) + } else if !s.Items.Type.Empty() && (s.Items.Properties == nil || len(s.Items.Properties) == 0) { + // inline object for array of untyped items + typ = protobuf.ListValueType + c.addImportForType(typ.Name()) + } else if !s.Items.Type.Empty() && len(s.Items.Properties) > 0 { + // inline object for array of typed items + baseFieldName := camelCase(name) + typ = c.createListWrapper(name, rawName, baseFieldName, s) + // finally, make sure that this type is registered, if need be. + c.addType(typ) + subtyp, err := c.compileSchema(name, s.Items) + if err == nil { + c.addType(subtyp) + } + } else { + return nil, errors.Errorf(`An array for map types must specify a reference or an object`) + } + } else { + typ, err = c.getType(s.Type.First()) + if err != nil { + return nil, errors.Wrapf(err, `failed to get type %s`, s.Type) + } } default: var err error @@ -501,7 +538,6 @@ func (c *compileCtx) compileMap(name string, s *openapi.Schema) (protobuf.Type, } } - // Note: Map of arrays is not currently supported. return protobuf.NewMap(protobuf.StringType, typ), nil } @@ -534,7 +570,6 @@ func (c *compileCtx) compileSchema(name string, s *openapi.Schema) (protobuf.Typ } return m, nil } - rawName := name name = camelCase(name) // could be a builtin... try as-is once, then the camel cased @@ -555,7 +590,7 @@ func (c *compileCtx) compileSchema(name string, s *openapi.Schema) (protobuf.Typ switch { case s.Type.Empty() || s.Type.Contains("object"): if ap := s.AdditionalProperties; ap != nil && !ap.IsNil() { - return c.compileMap(name, ap) + return c.compileMap(name, strings.TrimSuffix(rawName, "Message"), ap) } m := protobuf.NewMessage(name) @@ -833,6 +868,15 @@ func (c *compileCtx) parent() protobuf.Container { return c.parents[l-1] } +func (c *compileCtx) grandParent() protobuf.Container { + switch len(c.parents) { + case 0: + return c.pkg + default: + return c.parents[0] + } +} + // adds new type. dedupes, in case of multiple addition func (c *compileCtx) addType(t protobuf.Type) { c.addTypeToParent(t, c.parent()) @@ -854,6 +898,22 @@ func (c *compileCtx) addTypeToParent(t protobuf.Type, p protobuf.Container) { } } + // hack alert - check for duplicates + // I couldn't figure out how to stop map list value wrappers from being specified more than once. + // This is generalized here based on the type hierarchy to prevent duplicates of all messages. + parentNames := func(vs []protobuf.Container) []string { + vsm := make([]string, len(vs)) + for i, v := range vs { + vsm[i] = v.Name() + } + return vsm + }(c.parents) + key := strings.Trim(strings.Join(parentNames, "#"), "[]") + "#" + t.Name() + if _, ok := c.messageNames[key]; ok { + return + } + c.messageNames[key] = true + m, ok := c.types[p] if !ok { m = map[protobuf.Type]struct{}{} @@ -903,9 +963,26 @@ func (c *compileCtx) compilePaths(paths map[string]*openapi.Path) error { return nil } +func (c *compileCtx) createListWrapper(name string, rawName string, baseFieldName string, s *openapi.Schema) protobuf.Type { + // we need to construct a new statically typed wrapper message that contains a repeated list of items + // referenced by the spec + mapValueName := strings.TrimSuffix(name, "Message") + "List" + m := protobuf.NewMessage(mapValueName) + f := protobuf.NewField(protobuf.NewMessage(baseFieldName), rawName, 1) + f.SetRepeated(true) + if v := s.Description; len(v) > 0 { + f.SetComment(v) + } + m.AddField(f) + m.SetComment("automatically generated wrapper for a list of " + baseFieldName + " items") + return m +} + func mergeParameters(p1, p2 openapi.Parameters) openapi.Parameters { var out openapi.Parameters out = append(out, p1...) out = append(out, p2...) return out } + + diff --git a/compiler/interface.go b/compiler/interface.go index e589e90..0a087f2 100644 --- a/compiler/interface.go +++ b/compiler/interface.go @@ -18,6 +18,8 @@ type Option = option.Option type compileCtx struct { annotate bool + skipRpcs bool + namespaceEnums bool definitions map[string]protobuf.Type externalDefinitions map[string]map[string]protobuf.Type imports map[string]struct{} @@ -29,4 +31,5 @@ type compileCtx struct { service *protobuf.Service types map[protobuf.Container]map[protobuf.Type]struct{} unfulfilledRefs map[string]struct{} + messageNames map[string]bool } diff --git a/compiler/options.go b/compiler/options.go index 2082842..c58ec7e 100644 --- a/compiler/options.go +++ b/compiler/options.go @@ -3,7 +3,9 @@ package compiler import "github.com/NYTimes/openapi2proto/internal/option" const ( - optkeyAnnotation = "annotation" + optkeyAnnotation = "annotation" + optkeySkipRpcs = "skip-rpcs" + optkeyPrefixEnums = "namespace-enums" ) // WithAnnotation creates a new Option to specify if we should add @@ -11,3 +13,13 @@ const ( func WithAnnotation(b bool) Option { return option.New(optkeyAnnotation, b) } + +// WithSkipRpcs creates a new Option to specify if we should +// generate services and rpcs in addition to messages +func WithSkipRpcs(b bool) Option { + return option.New(optkeySkipRpcs, b) +} + +func WithPrefixEnums(b bool) Option { + return option.New(optkeyPrefixEnums, b) +} diff --git a/fixtures/refs.json b/fixtures/refs.json index b334bf7..4233772 100644 --- a/fixtures/refs.json +++ b/fixtures/refs.json @@ -5,23 +5,58 @@ }, "definitions": { "TestModel": { - "type": "object", - "properties": { - "test_map_object": { - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/TestModel" - }, - "x-proto-tag": 11 - }, - "test_map_scalar": { - "type": "object", - "additionalProperties": { - "type": "string" - }, - "x-proto-tag": 12 + "type": "object", + "properties": { + "test_map_object": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/TestModel" + }, + "x-proto-tag": 11 + }, + "test_map_scalar": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "x-proto-tag": 12 + }, + "test_map_array": { + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "$ref": "#/definitions/TestModel" } + }, + "x-proto-tag": 13 + }, + "test_map_array_untyped_fields": { + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "integer" + } + } + } + }, + "x-proto-tag": 15 + }, + "test_map_array_untyped": { + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "type": "object" + } + }, + "x-proto-tag": 14 } + } } } } diff --git a/fixtures/refs.proto b/fixtures/refs.proto index e9f963e..c9ae98b 100644 --- a/fixtures/refs.proto +++ b/fixtures/refs.proto @@ -2,7 +2,26 @@ syntax = "proto3"; package refs; +import "google/protobuf/struct.proto"; + +// automatically generated wrapper for a list of TestModel items +message TestMapArrayList { + repeated TestModel test_map_array = 1; +} + message TestModel { + // automatically generated wrapper for a list of TestMapArrayUntypedFieldsMessage items + message TestMapArrayUntypedFieldsList { + repeated TestMapArrayUntypedFieldsMessage test_map_array_untyped_fields = 1; + } + + message TestMapArrayUntypedFieldsMessage { + int32 id = 1; + } + map test_map_object = 11; map test_map_scalar = 12; + map test_map_array = 13; + map test_map_array_untyped = 14; + map test_map_array_untyped_fields = 15; } diff --git a/fixtures/refs.yaml b/fixtures/refs.yaml index de6a6e3..bf0a7f7 100644 --- a/fixtures/refs.yaml +++ b/fixtures/refs.yaml @@ -15,3 +15,27 @@ definitions: additionalProperties: type: string x-proto-tag: 12 + test_map_array: + type: object + additionalProperties: + type: array + items: + $ref: '#/definitions/TestModel' + x-proto-tag: 13 + test_map_array_untyped: + type: object + additionalProperties: + type: array + items: + type: object + x-proto-tag: 14 + test_map_array_untyped_fields: + type: object + additionalProperties: + type: array + items: + type: object + properties: + id: + type: integer + x-proto-tag: 15 diff --git a/protobuf/interface.go b/protobuf/interface.go index 1f64e11..87ba7ae 100644 --- a/protobuf/interface.go +++ b/protobuf/interface.go @@ -40,6 +40,11 @@ var ( StringValueType = NewMessage("google.protobuf.StringValue") ) +// list type +var ( + ListValueType = NewMessage("google.protobuf.ListValue") +) + var ( emptyMessage = NewMessage("google.protobuf.Empty") )