Skip to content

Commit

Permalink
✨ feat #302: proto to kcl_schema (#307)
Browse files Browse the repository at this point in the history
* ✨ feat: proto to kcl_schema

Signed-off-by: xiaok29 <1526783667@qq.com>

* fix:add bool float type

Signed-off-by: xiaok29 <1526783667@qq.com>

* fix:line breaks on different platforms

Signed-off-by: xiaok29 <1526783667@qq.com>

* fix:line breaks on different platforms

Signed-off-by: xiaok29 <1526783667@qq.com>

* perf:use writeString splicing

Signed-off-by: xiaok29 <1526783667@qq.com>

* Update pkg/tools/gen/genkcl_proto.go

Co-authored-by: Peefy <xpf6677@163.com>
Signed-off-by: xiaok29 <1526783667@qq.com>

* feat:read enum type

Signed-off-by: xiaok29 <1526783667@qq.com>

---------

Signed-off-by: xiaok29 <1526783667@qq.com>
Co-authored-by: Peefy <xpf6677@163.com>
  • Loading branch information
XiaoK29 and Peefy authored May 21, 2024
1 parent 0d1338f commit e160f6c
Show file tree
Hide file tree
Showing 8 changed files with 328 additions and 10 deletions.
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ require (
github.com/docker/go-metrics v0.0.1 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/dominikbraun/graph v0.23.0 // indirect
github.com/emicklei/proto v1.13.2
github.com/emirpasic/gods v1.18.1 // indirect
github.com/fatih/color v1.10.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -302,6 +302,8 @@ github.com/dominikbraun/graph v0.23.0 h1:TdZB4pPqCLFxYhdyMFb1TBdFxp8XLcJfTTBQucV
github.com/dominikbraun/graph v0.23.0/go.mod h1:yOjYyogZLY1LSG9E33JWZJiq5k83Qy2C6POAuiViluc=
github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU=
github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
github.com/emicklei/proto v1.13.2 h1:z/etSFO3uyXeuEsVPzfl56WNgzcvIr42aQazXaQmFZY=
github.com/emicklei/proto v1.13.2/go.mod h1:rn1FgRS/FANiZdD2djyH7TMA9jdRDcYQ9IEN9yvjX0A=
github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
Expand Down
24 changes: 14 additions & 10 deletions pkg/tools/gen/genkcl.go
Original file line number Diff line number Diff line change
@@ -1,22 +1,22 @@
package gen

import (
"encoding/json"
"errors"
"fmt"
"io"
"path/filepath"
"regexp"
"strconv"
"strings"

"github.com/goccy/go-yaml"
"kcl-lang.io/kcl-go/pkg/logger"
)

type GenKclOptions struct {
Mode Mode
ParseFromTag bool
CastingOption castingOption
Mode Mode
ParseFromTag bool
CastingOption castingOption
UseIntegersForNumbers bool
}

// Mode is the mode of kcl schema code generation.
Expand All @@ -29,6 +29,7 @@ const (
ModeTerraformSchema
ModeJson
ModeYaml
ModeProto
)

type kclGenerator struct {
Expand Down Expand Up @@ -56,9 +57,8 @@ func (k *kclGenerator) GenSchema(w io.Writer, filename string, src interface{})
return err
}
codeStr := string(code)
var i interface{}
switch {
case json.Unmarshal(code, &i) == nil:
switch filepath.Ext(filename) {
case ".json":
switch {
case strings.Contains(codeStr, "$schema"):
k.opts.Mode = ModeJsonSchema
Expand All @@ -67,10 +67,12 @@ func (k *kclGenerator) GenSchema(w io.Writer, filename string, src interface{})
default:
k.opts.Mode = ModeJson
}
case yaml.Unmarshal(code, &i) == nil:
case ".yaml", "yml":
k.opts.Mode = ModeYaml
case strings.Contains(codeStr, "package "):
case ".go":
k.opts.Mode = ModeGoStruct
case ".proto":
k.opts.Mode = ModeProto
default:
return errors.New("failed to detect mode")
}
Expand All @@ -87,6 +89,8 @@ func (k *kclGenerator) GenSchema(w io.Writer, filename string, src interface{})
return k.genKclFromJsonData(w, filename, src)
case ModeYaml:
return k.genKclFromYaml(w, filename, src)
case ModeProto:
return k.genKclFromProtoData(w, filename, src)
default:
return errors.New("unknown mode")
}
Expand Down
161 changes: 161 additions & 0 deletions pkg/tools/gen/genkcl_proto.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,161 @@
package gen

import (
"bufio"
"bytes"
"fmt"
"io"
"runtime"
"strconv"
"strings"

"github.com/emicklei/proto"
)

var defaultFieldTypeMap = map[string]string{
"uint32": "int",
"uint64": "int",
"int32": "int",
"int64": "int",
"sint32": "int",
"sint64": "int",
"string": "str",
"google.protobuf.Any": "any",
"bool": "bool",
"float": "float",
"double": "float",
}

// genKclFromProtoData
func (k *kclGenerator) genKclFromProtoData(w io.Writer, filename string, src interface{}) error {
lineBreak := "\n"
if runtime.GOOS == "windows" {
lineBreak = "\r\n"
}

code, err := readSource(filename, src)
if err != nil {
return err
}

parser := proto.NewParser(bytes.NewBuffer(code))
definitions, err := parser.Parse()
if err != nil {
return fmt.Errorf(`error parsing proto file %v: %v`, filename, err)
}

fieldTypeMap := k.genFieldTypeMap(definitions)
builder := bufio.NewWriter(w)
for _, definition := range definitions.Elements {
message, ok := definition.(*proto.Message)
if !ok {
continue
}

builder.WriteString("schema ")
builder.WriteString(message.Name)
builder.WriteString(":")
builder.WriteString(lineBreak)

for _, element := range message.Elements {
switch field := element.(type) {
case *proto.NormalField:
builder.WriteString(" ")
builder.WriteString(field.Name)
if field.Optional {
builder.WriteString("?")
}
builder.WriteString(": ")

if field.Repeated {
builder.WriteString("[")
}

fieldType, err := getFieldType(fieldTypeMap, field.Type)
if err != nil {
return err
}
builder.WriteString(fieldType)

if field.Repeated {
builder.WriteString("]")
}
builder.WriteString(lineBreak)

case *proto.MapField:
builder.WriteString(" ")
builder.WriteString(field.Name)
builder.WriteString(": {")
keyType, err := getFieldType(fieldTypeMap, field.KeyType)
if err != nil {
return err
}
builder.WriteString(keyType)
builder.WriteString(":")
fieldType, err := getFieldType(fieldTypeMap, field.Type)
if err != nil {
return err
}
builder.WriteString(fieldType)
builder.WriteString("}")
builder.WriteString(lineBreak)
}
}

builder.WriteString(lineBreak)
}

if err = builder.Flush(); err != nil {
return err
}

return nil
}

// GenFieldTypeMap
func (k *kclGenerator) genFieldTypeMap(definitions *proto.Proto) map[string]string {
fieldTypeMap := make(map[string]string)
for key, value := range defaultFieldTypeMap {
fieldTypeMap[key] = value
}

for _, definition := range definitions.Elements {
switch visitee := definition.(type) {
case *proto.Message:
fieldTypeMap[visitee.Name] = visitee.Name
case *proto.Enum:
var builder strings.Builder
elementsLen := len(visitee.Elements) - 1
for i, e := range visitee.Elements {
v, ok := e.(*proto.EnumField)
if !ok {
continue
}

value := fmt.Sprintf(`"%v"`, v.Name)
if k.opts.UseIntegersForNumbers {
value = strconv.Itoa(v.Integer)
}

builder.WriteString(value)
if elementsLen > i {
builder.WriteString(` | `)
}

fieldTypeMap[v.Name] = v.Name
}
fieldTypeMap[visitee.Name] = builder.String()
}
}

return fieldTypeMap
}

func getFieldType(fieldTypeMap map[string]string, fieldType string) (string, error) {
value, ok := fieldTypeMap[fieldType]
if !ok {
return "", fmt.Errorf(`this "%v" is not currently supported`, fieldType)
}

return value, nil
}
24 changes: 24 additions & 0 deletions pkg/tools/gen/genkcl_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,30 @@ func TestGenKclFromMultipleResourceYaml(t *testing.T) {
}
}

func TestGenKclFromProto(t *testing.T) {
t.Run("not UseIntegersForNumbers", func(t *testing.T) {
var buf bytes.Buffer
err := GenKcl(&buf, `./testdata/proto/proto2kcl.proto`, nil, &GenKclOptions{UseIntegersForNumbers: false})
if err != nil {
t.Fatal(err)
}

b, _ := os.ReadFile("./testdata/proto/proto2kcl.k")
assert2.Equal(t, string(b), buf.String())
})

t.Run("UseIntegersForNumbers", func(t *testing.T) {
var buf bytes.Buffer
err := GenKcl(&buf, `./testdata/proto/proto2kcl.proto`, nil, &GenKclOptions{UseIntegersForNumbers: true})
if err != nil {
t.Fatal(err)
}

b, _ := os.ReadFile("./testdata/proto/proto2kcl_for_num.k")
assert2.Equal(t, string(b), buf.String())
})
}

type TestData = data

func TestGenKclFromJsonAndImports(t *testing.T) {
Expand Down
36 changes: 36 additions & 0 deletions pkg/tools/gen/testdata/proto/proto2kcl.k
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
schema Person:
name: str
age: int
friends: [str]
movies: {str:Movie}
employee?: employee
company: Company
star_int?: int
star_map: {str:str}
inter: any

schema Movie:
desc: str
size: int
kind: str
unknown1: any
unknown2: any

schema employee:
name: str
age: int
friends: [str]
movies: {str:Movie}
bank_card: int
nationality: str
salary: float
age_double: float
is_married: bool
gender: "unknown" | "male" | "female"
gender_opt?: "unknown" | "male" | "female"

schema Company:
name: str
employees: [employee]
persons?: Person

54 changes: 54 additions & 0 deletions pkg/tools/gen/testdata/proto/proto2kcl.proto
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
syntax = "proto3";
package test;
option go_package = "test";

import "google/protobuf/any.proto";


message Person{
string name = 1;
int64 age = 2;
repeated string friends = 3;
map<string,Movie> movies = 4;
optional employee employee = 6;
Company company = 7;
optional int32 star_int = 8;
map<string,string> star_map = 9;
google.protobuf.Any inter = 10;
}

message Movie{
string desc = 1;
int32 size = 2;
string kind = 3;
google.protobuf.Any unknown1 = 4;
google.protobuf.Any unknown2 = 5;
}


message employee{
string name = 1;
uint32 age = 2;
repeated string friends = 3;
map<string,Movie> movies = 4;
int32 bank_card = 5;
string nationality = 6;
float salary = 7;
double age_double = 8;
bool is_married = 9;
gender gender = 10;
optional gender gender_opt = 10;
}


message Company{
string name = 1;
repeated employee employees = 2;
optional Person persons = 3;
}

enum gender{
unknown = 0;
male = 1;
female = 2;
}
Loading

0 comments on commit e160f6c

Please sign in to comment.